def getFold(fold=0,
fname_in='trainFolds.csv',
fnames=['CTs.csv', 'Fleischner.csv', 'Nodules.csv'],
prefix_in='train',
prefix_out='',
excludeFold=True):
if not prefix_out:
prefix_out = 'fold{}'.format(fold) # eg. fold0
#Get fold lnds
nodules = readCsv(fname_in)
header = nodules[0]
lines = nodules[1:]
foldind = header.index('Fold{}'.format(fold)) # get fold idx from file
foldlnd = [l[foldind] for l in lines if len(l) > foldind
] # select correct lnd number except with missing data
for fname in fnames: # loop thru filetypes
lines = readCsv(prefix_in + fname)
header = lines[0]
lines = lines[1:]
lndind = header.index('LNDbID')
if not excludeFold:
lines = [l for l in lines if l[lndind] in foldlnd]
else:
lines = [l for l in lines if not l[lndind] in foldlnd]
#Save to csv
writeCsv(prefix_out + fname, [header] + lines)
def getFold(fold=0, fname_in='trainFolds.csv',
fnames=['CTs.csv', 'Fleischner.csv', 'Nodules.csv'],
prefix_in='train', prefix_out='',
excludeFold=False):
if not prefix_out:
prefix_out = 'fold{}'.format(fold)
# Get fold lnds
nodules = readCsv(fname_in)
header = nodules[0]
lines = nodules[1:]
foldind = header.index('Fold{}'.format(fold))
foldlnd = [l[foldind] for l in lines if len(l) > foldind]
for fname in fnames:
lines = readCsv(prefix_in + fname)
header = lines[0]
lines = lines[1:]
lndind = header.index('LNDbID')
if not excludeFold:
lines = [l for l in lines if l[lndind] in foldlnd]
else:
lines = [l for l in lines if not l[lndind] in foldlnd]
# Save to csv
writeCsv(prefix_out + fname, [header] + lines)
def assessWithoutDist(self):
print "reading data..."
pictures = readCsv(self.dataBaseUrl + "data", self.totalNum)
dao = ImageDao()
imgs = dao.getAll()
typeDict = {}
for img in imgs:
typeDict[img.imgId] = img.imgType
print "training..."
trainSet = pictures[:self.trainNum]
self.knn.train(trainSet)
testSet = pictures[self.trainNum:self.totalNum]
accuracyList = []
heads = ['distance', 'accuracy', 'averageK']
print "predicting..."
for d in range(2000, 4000, 20):
accuracy, avgK = self.knn.predictForManyWithDist(
testSet, self.trainNum, d, typeDict)
item = [d, accuracy, avgK]
accuracyList.append(item)
print "distance:%d accuracy:%f%% averageK:%f" % (
d, accuracy * 100, avgK)
saveCsv(self.resultBasePath + 'assessDist_Radius10_5000-1000.csv',
heads, accuracyList)
def pair(fileName):
try:
datas = readCsv(fileName)
except FileNotFoundError as e:
print(e)
exit()
groupedBy = datas.groupby('Hogwarts House')
fig, axes = plt.subplots(13, 13, figsize=(15, 10))
fig.subplots_adjust(hspace=0.1, wspace=0.1)
for ax in axes:
for a in ax:
a.set_xticklabels([])
a.set_yticklabels([])
a.tick_params(axis='both', width=0)
for i in range(len(courses)):
groupedBy[courses[i]].plot(kind='hist', alpha=0.5, ax=axes[i][i])
axes[i][i].set_xlabel(courses[i].replace(' ', '\n'), fontsize=8)
axes[i][i].set_ylabel(courses[i].replace(' ', '\n'), fontsize=8)
for i, xi in enumerate(courses):
for j, yi in enumerate(courses):
if j < i:
axes[i][j].remove()
continue
if xi == yi:
continue
ravenclowX = groupedBy.get_group('Ravenclaw')[xi]
slytherinX = groupedBy.get_group('Slytherin')[xi]
gryffindorX = groupedBy.get_group('Gryffindor')[xi]
hufflepuffX = groupedBy.get_group('Hufflepuff')[xi]
ravenclowY = groupedBy.get_group('Ravenclaw')[yi]
slytherinY = groupedBy.get_group('Slytherin')[yi]
gryffindorY = groupedBy.get_group('Gryffindor')[yi]
hufflepuffY = groupedBy.get_group('Hufflepuff')[yi]
axes[i][j].plot(ravenclowX,
ravenclowY,
'v',
markersize=.5,
color='blue')
axes[i][j].plot(slytherinX,
slytherinY,
'^',
markersize=.5,
color='green')
axes[i][j].plot(gryffindorX,
gryffindorY,
'o',
markersize=.5,
color='red')
axes[i][j].plot(hufflepuffX,
hufflepuffY,
's',
markersize=.5,
color='yellow')
plt.show()
def assess(self, k=None):
if not k:
k = self.k
print "reading data..."
pictures = readCsv(self.dataBaseUrl + "data", self.totalNum)
dao = ImageDao()
imgs = dao.getAll()
typeDict = {}
for img in imgs:
typeDict[img.imgId] = img.imgType
print "training..."
trainSet = pictures[:self.trainNum]
self.knn.train(trainSet)
testSet = pictures[self.trainNum:self.totalNum]
zerNp = np.zeros([k, self.testNum])
testLabel = np.arange(self.trainNum, self.totalNum)
for i in range(len(testLabel)):
testLabel[i] = typeDict[str(testLabel[i]).zfill(5)]
testLabel = (zerNp + testLabel).astype('int').T
print "predicting..."
accuracy, avgCriDist = self.knn.predictForManyWithK(
testSet, testLabel, k, typeDict)
print "accuracy:%f%% averageCriticalDist:%f" % (accuracy * 100,
avgCriDist)
def getTextures(count):
csvLines = utils.readCsv("../trainset_csv/trainNodules_gt.csv")
textures = [row[-1] for row in csvLines]
# delete 1st element
del textures[0]
if count != -1:
del textures[count:]
return textures
def getMaskVolumes():
csvLines = utils.readCsv("../trainset_csv/trainNodules_gt.csv")
last_ID = 0
scan = 0
spacing = 0
origin = 0
maskVolumesList = []
# ignore header
for line in csvLines[1:]:
# get image of this patient only one time (there are repeated patient ids)
current_ID = line[0]
if last_ID != current_ID:
print(getFileID(current_ID))
scan, spacing, origin, _ = utils.readMhd(
'../LNDb dataset/dataset/LNDb-' + getFileID(current_ID) +
'.mhd')
spacing = [float(spacing[i]) for i in range(3)]
# find the coordinates of the current nodule (it is done for every line of the csv)
finding_coords = line[4:7]
nodule_x = (float(finding_coords[0]) - float(origin[0])) / float(
spacing[0])
nodule_y = (float(finding_coords[1]) - float(origin[1])) / float(
spacing[1])
nodule_z = (float(finding_coords[2]) - float(origin[2])) / float(
spacing[2])
real_coords = [nodule_x, nodule_y, nodule_z]
# get a mask for the image of this patient (from one of the radiologists that found the current nodule)
radiologists = line[
1] # list of radiologists that found the current nodule
radId = str(
radiologists[0]
) # always choose the mask from the first radiologist in the list
mask, _, _, _ = utils.readMhd('../LNDb dataset/masks/LNDb-' +
getFileID(current_ID) + '_rad' + radId +
'.mhd')
# extract mini cube of the current nodule on the masked scan
mask_volume = utils.extractCube(mask,
spacing,
real_coords,
cube_size=80)
# add mask volumes to the list
maskVolumesList.append(mask_volume)
last_ID = current_ID
return maskVolumesList
def assessFeaWithBlurWithoutK(self):
print "reading neaFeaData..."
netFea = readCsv(self.dataBaseUrl + "netFea", self.totalNum) * 256 * 40
print "reading data..."
pictures = readCsv(self.dataBaseUrl + "data", self.totalNum)
pictures = np.append(pictures, netFea, axis=1)
print pictures.shape
dao = ImageDao()
imgs = dao.getAll()
typeDict = {}
for img in imgs:
typeDict[img.imgId] = img.imgType
print "training..."
trainSet = pictures[:self.trainNum]
self.knn.train(trainSet)
testSet = pictures[self.trainNum:self.totalNum]
accuracyList = []
heads = ['k', 'accuracy', 'averageCriticalDist']
print "predicting..."
for k in range(1, 101):
zerNp = np.zeros([k, self.testNum])
testLabel = np.arange(self.trainNum, self.totalNum)
for i in range(len(testLabel)):
testLabel[i] = typeDict[str(testLabel[i]).zfill(5)]
testLabel = (zerNp + testLabel).astype('int').T
accuracy, avgCriDist = self.knn.predictForManyWithK(
testSet, testLabel, k, typeDict)
item = [k, accuracy, avgCriDist]
accuracyList.append(item)
print "k:%d accuracy:%f%% averageCriticalDist:%f" % (
k, accuracy * 100, avgCriDist)
saveCsv(self.resultBasePath + 'assessFeaWithBlurWithoutK.csv', heads,
accuracyList)
def createCube(cubeSize=80):
csvLines = readCsv("csv/trainSet.csv")
header = csvLines[0]
nodules = csvLines[1:]
for i, n in tqdm(enumerate(nodules[:1])):
x = int(n[header.index("x")])
y = int(n[header.index("y")])
z = int(n[header.index("z")])
lnd = int(n[header.index("LNDbID")])
ctr = np.array([x, y, z])
[scan, spacing, _, _] = readMhd('data/LNDb-{:04}.mhd'.format(lnd))
cube = extractCube(scan, spacing, ctr, cubeSize)
np.save("cube/{:0}.npy".format(i), cube)
def assessWithoutRadius(self, k=None):
if not k:
k = self.k
accuracyList = []
heads = ['radius', 'accuracy', 'averageCriticalDist']
for radius in range(5, 15):
print radius, ':'
print "blurring ..."
data = MyData()
data.saveCsvWithGaussianBlur(radius=radius)
dao = ImageDao()
imgs = dao.getAll()
typeDict = {}
for img in imgs:
typeDict[img.imgId] = img.imgType
zerNp = np.zeros([k, self.testNum])
testLabel = np.arange(self.trainNum, self.totalNum)
for i in range(len(testLabel)):
testLabel[i] = typeDict[str(testLabel[i]).zfill(5)]
testLabel = (zerNp + testLabel).astype('int').T
pictures = readCsv(self.dataBaseUrl + "data", self.totalNum)
print "training..."
trainSet = pictures[:self.trainNum]
self.knn.train(trainSet)
testSet = pictures[self.trainNum:self.totalNum]
print "predicting..."
accuracy, avgCriDist = self.knn.predictForManyWithK(
testSet, testLabel, k, typeDict)
item = [radius, accuracy, avgCriDist]
accuracyList.append(item)
print "k:%d radius:%f accuracy:%f%% averageCriticalDist:%f" % (
k, radius, accuracy * 100, avgCriDist)
saveCsv(self.resultBasePath + 'assessRadiusK' + str(k) + '.csv', heads,
accuracyList)
def scatter(fileName):
try:
datas = readCsv(fileName)
except FileNotFoundError as e:
print(e)
exit()
groupedBy = datas.groupby('Hogwarts House')
fig, axes = plt.subplots(13, 13, figsize=(15, 10))
fig.subplots_adjust(hspace=0, wspace=0)
i = 0
for ax, course in zip(axes, courses):
ax[i].text(0.5, 0.5, course.replace(' ', '\n'), ha='center', va='center', fontsize=8)
for a in ax:
a.set_xticklabels([])
a.set_yticklabels([])
a.tick_params(axis='both', width=0)
i += 1
for i, xi in enumerate(courses):
for j, yi in enumerate(courses):
if xi == yi :
continue
ravenclowX = groupedBy.get_group('Ravenclaw')[xi]
slytherinX = groupedBy.get_group('Slytherin')[xi]
gryffindorX = groupedBy.get_group('Gryffindor')[xi]
hufflepuffX = groupedBy.get_group('Hufflepuff')[xi]
ravenclowY = groupedBy.get_group('Ravenclaw')[yi]
slytherinY = groupedBy.get_group('Slytherin')[yi]
gryffindorY = groupedBy.get_group('Gryffindor')[yi]
hufflepuffY = groupedBy.get_group('Hufflepuff')[yi]
axes[i][j].plot(ravenclowX, ravenclowY, 'v', markersize=.5, color='blue')
axes[i][j].plot(slytherinX, slytherinY, '^', markersize=.5, color='green')
axes[i][j].plot(gryffindorX, gryffindorY, 'o', markersize=.5, color='red')
axes[i][j].plot(hufflepuffX, hufflepuffY, 's', markersize=.5, color='yellow')
plt.show()
def histogram(fileName):
try:
datas = readCsv(fileName)
except FileNotFoundError as e:
print(e)
exit(1)
plt.figure(figsize=(8, 6))
groupedBy = datas.groupby('Hogwarts House')
for course in courses:
groupedBy[course].plot(kind='hist', alpha=0.5)
plt.title("Histogram of notes frequency by House \n for " + course +
" course")
plt.legend(loc='upper left')
print("Notes frequency by House for " + course)
print(
"Close graphical windows to see next histogram or press Ctrl + W \n"
)
plt.show()
def describe(fileName):
datas = readCsv(fileName)
datas = dropColumns(datas, ["Index", "Hogwarts House", "First Name",
"Last Name", "Birthday", "Best Hand"])
result = pd.DataFrame(
columns=datas.columns,
index=["Count","Mean","Std","Min","25%","50%","75%","Max"
]
)
result.iloc[0] = count(datas)
result.iloc[1] = mean(datas)
result.iloc[2] = std(datas)
result.iloc[3] = minimum(datas)
result.iloc[4] = quantile(datas, 0.25)
result.iloc[5] = quantile(datas, 0.50)
result.iloc[6] = quantile(datas, 0.75)
result.iloc[7] = maximum(datas)
print(result)
def generate_data(data_file, fold, data_path='../data/', is_train=True):
lines = readCsv(data_file)
header = lines[0]
lines = lines[1:]
consolidate_centers = {}
consolidate_labels = {}
# loop through lines, collate labels, centers by images
for line in lines:
process_line(header, line, consolidate_centers, consolidate_labels)
# load and save examples (subimages) and labels (textures)
examples = []
labels = []
for Id, centers in consolidate_centers.items():
img_name = 'LNDb-{:04}.mhd'.format(int(Id))
examples += load_image(os.path.join(data_path, img_name), centers)
labels += consolidate_labels[Id]
examples = np.array(examples)
labels = np.array(labels)
# now the labels and examples should be in order
# so we save as a pickle file (serialized file) to be loaded by ML model
data = ('train' if is_train else 'val') + \
'_data_except_{}.npy'.format(fold)
texture_file = ('train' if is_train else 'val') + \
'_labels_except_{}.npy'.format(fold)
DATAFILE = open(data, mode='wb')
pickle.dump(examples, DATAFILE)
DATAFILE.close()
LABELFILE = open(texture_file, mode='wb')
pickle.dump(labels, LABELFILE)
LABELFILE.close()
def getCubes(cubeSize):
csvLines = utils.readCsv("../trainset_csv/trainNodules_gt.csv")
last_ID = 0
scan = 0
spacing = 0
origin = 0
cubeList = []
textures = [row[-1] for row in csvLines]
# delete 1st element (header)
del textures[0]
# ignore header
for line in csvLines[1:]:
current_ID = line[0]
if last_ID != current_ID:
print(getFileID(current_ID))
scan,spacing,origin,_ = utils.readMhd('../LNDb dataset/dataset/LNDb-' + getFileID(current_ID) + '.mhd')
spacing = [float(spacing[i]) for i in range(3)]
finding_coords = line[4:7]
nodule_x = (float(finding_coords[0]) - float(origin[0])) / float(spacing[0])
nodule_y = (float(finding_coords[1]) - float(origin[1])) / float(spacing[1])
nodule_z = (float(finding_coords[2]) - float(origin[2])) / float(spacing[2])
real_coords = [nodule_x, nodule_y, nodule_z]
scan_cube = utils.extractCube(scan, spacing, real_coords, cube_size=cubeSize)
cubeList.append(scan_cube)
# nodule_coords
last_ID = current_ID
return cubeList, textures
import numpy as np
import copy
from matplotlib import pyplot as plt
from utils import readMhd, readCsv, getImgWorldTransfMats, convertToImgCoord, extractCube
from readNoduleList import nodEqDiam
dispFlag = False
# Read nodules csv
csvlines = readCsv('trainNodules_gt.csv')
header = csvlines[0]
nodules = csvlines[1:]
lndloaded = -1
for n in nodules:
vol = float(n[header.index('Volume')])
if nodEqDiam(vol) > 3: #only get nodule cubes for nodules>3mm
ctr = np.array([
float(n[header.index('x')]),
float(n[header.index('y')]),
float(n[header.index('z')])
])
lnd = int(n[header.index('LNDbID')])
rads = list(map(int, list(n[header.index('RadID')].split(','))))
radfindings = list(
map(int, list(n[header.index('RadFindingID')].split(','))))
finding = int(n[header.index('FindingID')])
print(lnd, finding, rads, radfindings)
# Read scan
import numpy as np from sklearn import svm from sklearn.metrics import mean_squared_error import utils # read csv data X, y = utils.readCsv() # train svm clf = svm.SVR(C=100000000, gamma=0.01, epsilon=200000, kernel='rbf') clf.fit(X, y) # evaluate fit on training data y_pred = clf.predict(X) score = np.sqrt(mean_squared_error(y, y_pred)) print "RMSE: %s" % score # # Output # RMSE: 545770.525711 # store svm utils.writeSVM(clf)
tex = 0
nodules.append([
int(n[lndind][0]),
','.join([str(int(r))
for r in n[radind]]), #list radiologist IDs
','.join([str(int(f)) for f in n[fndind]
]), #list radiologist finding's IDs
ind + 1, # new finding ID
np.mean(n[xind]), #centroid is the average of centroids
np.mean(n[yind]),
np.mean(n[zind]),
agrlvl, # number of radiologists that annotated the finding (0 if non-nodule)
nod,
vol,
tex
])
if verb:
for n in nodules:
print(n)
return nodules
if __name__ == "__main__":
# Merge nodules from train set
prefix = 'train'
fname_gtNodulesFleischner = '{}Nodules.csv'.format(prefix)
gtNodules = readCsv(fname_gtNodulesFleischner)
for line in gtNodules:
print(line)
gtNodules = joinNodules(gtNodules)
writeCsv('{}Nodules_gt.csv'.format(prefix), gtNodules) #write to csv
a[1][i].imshow(np.reshape(encode_decode[i], (50, 50, 3)))
f.show()
plt.draw()
plt.show()
def predict(self):
saver = tf.train.Saver()
saver.restore(self.sess, "network/model/model.ckpt")
encode = self.sess.run(self.encoder_op,
feed_dict={self.X: self.dataSet})
size = encode.shape[0]
i = 1
while (i * 1000) < size:
np.savetxt('DataSet/netFea' + str(i - 1) + '.csv',
encode[(i - 1) * 1000:i * 1000],
delimiter=',')
i = i + 1
np.savetxt('DataSet/netFea' + str(i - 1) + '.csv',
encode[(i - 1) * 1000:size],
delimiter=',')
if __name__ == '__main__':
print "reading data..."
dataSet = readCsv("DataSetRaw/data", -1)
model = AutoEnc_Tensor(dataSet=dataSet)
# model.train()
# model.test()
model.predict()
import numpy as np
import copy
from matplotlib import pyplot as plt
from utils import readMhd, readCsv, getImgWorldTransfMats, convertToImgCoord, extractCube
from readNoduleList import nodEqDiam
import cv2 as cv
dispFlag = False
# Read nodules csv
csvlines = readCsv('../trainset_csv/trainNodules_gt.csv')
header = csvlines[0]
nodules = csvlines[1:]
# Descriptors Algorithm
orb = cv.ORB_create()
def computeDescriptors(scan_cube):
descriptors3d = []
for s in scan_cube:
# convert to grayscale
img = s.astype(np.float64)
if img.max() != img.min():
img = (img - img.min()) * (255.0 / (img.max() - img.min()))
img = img.astype(np.uint8)
# compute descriptors
kp, des = orb.detectAndCompute(img, None)
if des is not None:
for d in des:
print "initializing..."
imgBaseURL = "img/"
dataBaseUrl = "DataSet/"
upperK = 50
select = -1
k = 0
optimalDist = 2200
optimalLeastK = 12
dataNum = -1
radius = 10
print "reading data..."
data = MyData()
pictures = readCsv(dataBaseUrl + "data", dataNum)
print "training..."
knn = KNearestNeighbor()
knn.train(pictures)
dao = ImageDao()
imgs = dao.getAll()
typeDict = {}
for img in imgs:
typeDict[img.imgId] = img.imgUrl
# server.config['UPLOAD_FOLDER'] = os.getcwd()
@server.route('/', methods=['GET', 'POST'])
from model import NestedUNet
from sklearn.model_selection import KFold
from sklearn.metrics import jaccard_score
def count_params(model): #計算網路參數
return sum(p.numel() for p in model.parameters() if p.requires_grad)
if __name__ == "__main__":
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.manual_seed(10)
reader = readCsv('./train_valid_CTs.csv')
data_list = readCsv('./data_list_train_valid.csv')
reader = reader[1:]
data_list = np.array(data_list[1:])
tr_data = list()
tr_label = list()
for number in range(len(reader)): #讀取檔案製作訓練資料集[病人,幾張,3,Y,X]
[scan, spacing, origin, transfmat
] = readMhd('traindata/LNDb-{:04}.mhd'.format(int(reader[number][0])))
[mask, spacing, origin, transfmat] = readMhd(
'trainlabel_1c1t/LNDb-{:04}_w_Text_merged-mask.mhd'.format(
int(reader[number][0])))
for z in range(scan.shape[0]):
tr_data.append(scan[z])
tr_label.append(mask[z])
help="Possible values: hz, power, dur")
parser.add_argument('-out',
dest="OUTPUT_FILE",
default="../data/output/bird_sort_hz_mix.txt",
help="Output txt file")
args = parser.parse_args()
INPUT_SAMPLE_FILE = args.INPUT_SAMPLE_FILE
SORT_BY = args.SORT_BY
MIN_VALUE = args.MIN_VALUE
MAX_VALUE = args.MAX_VALUE
OUTPUT_FILE = args.OUTPUT_FILE
OVERLAP = 0.25
print("Reading data file...")
sampleData = readCsv(INPUT_SAMPLE_FILE)
print("Found %s rows in %s" % (len(sampleData), INPUT_SAMPLE_FILE))
# filter values
sampleData = [s for s in sampleData if MIN_VALUE <= s[SORT_BY] <= MAX_VALUE]
# sort values
sampleData = sorted(sampleData, key=lambda k: k[SORT_BY])
values = [s[SORT_BY] for s in sampleData]
print("%s range: [%s - %s]" % (SORT_BY, min(values), max(values)))
totalDur = sum([int(s["dur"] * (1.0 - OVERLAP)) for s in sampleData])
print("Total time: %s" %
time.strftime('%H:%M:%S', time.gmtime(totalDur / 1000)))
instructions = []
import numpy as np
from sklearn import svm
from sklearn.cross_validation import KFold
from sklearn.metrics import mean_squared_error
import utils
# read csv data
X, y = utils.readCsv()
# 10 folds cross validation svm training
rmse = []
kf = KFold(len(X), n_folds=10)
for train, test in kf:
clf = svm.SVR(C=100000000, gamma=0.01, epsilon=200000, kernel='rbf')
clf.fit(X[train], y[train])
y_pred = clf.predict(X[test])
score = np.sqrt(mean_squared_error(y[test], y_pred))
rmse.append(score)
print "RMSE: %s" % score
print "Min RMSE: %s\nMax RMSE: %s\nMean RMSE: %s" % (
np.min(rmse), np.max(rmse), np.mean(rmse))
# # Output
# RMSE: 464307.13222
# RMSE: 580966.456102
# RMSE: 519024.052988
# RMSE: 608102.489362
# RMSE: 606586.000127
# RMSE: 600282.364792
numpy.mean(numpy.sqrt(c_array[1]))
]
costs.append(cost)
print 'Training epoch %d, reconstruction cost ' % cost[0], numpy.mean(
cost[1]), ' jacobian norm ', cost[2]
heads = ['epoch', 'cost', 'jacobian']
saveCsv(fileName="layer_1Cost.csv", heads=heads, datas=costs)
end_time = timeit.default_timer()
training_time = (end_time - start_time)
print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((training_time) / 60.))
image = Image.fromarray(
tile_raster_images(X=ca.W.get_value(borrow=True).T,
img_shape=(50, 50 * 3),
tile_shape=(10, 10),
tile_spacing=(1, 1)))
image.save('cae_filters.png')
numpy.savetxt('layer_1W.csv', ca.W.get_value(borrow=True), delimiter=',')
os.chdir('../')
if __name__ == '__main__':
print "reading data..."
train_set_x = readCsv("DataSetRaw/data", 1000)
test_cA(train_set_x=train_set_x, n_hidden=100)
def calcNodTexClass(nodtex):
# Compute texture class from texture rating nodtex
texthr = [7 / 3, 11 / 3]
if isinstance(nodtex, float) or isinstance(nodtex, int):
if nodtex >= texthr[0] and nodtex <= texthr[1]:
vclass = 1
elif nodtex > texthr[1]:
vclass = 2
else:
vclass = 0
else: # numpy array
vclass = np.zeros(nodtex.shape)
vclass[np.bitwise_and(nodtex >= texthr[0], nodtex < texthr[1])] = 1
vclass[nodtex >= texthr[1]] = 2
return vclass
if __name__ == "__main__":
# Compute Fleischner score for all trainset nodules
fname_gtNodulesFleischner = 'trainNodules.csv'
gtNodules = readCsv(fname_gtNodulesFleischner)
gtNodules = joinNodules(gtNodules)
pdFleischner = calcFleischner(gtNodules)
# Compute Fleischner score for all predicted nodules (given volume and texture rating/class/probabilities)
fname_pdNodulesFleischner = 'predictedNodules.csv'
pdNodules = readCsv('input/' + fname_pdNodulesFleischner)
pdFleischner = calcFleischner(pdNodules)
val = [(epoch + 1), i, c]
costs.append(val)
# saver.save(self.sess,'network/model/model.ckpt')
heads = ['epoch', 'cost']
# saveCsv(fileName="network/netCost.csv", heads=heads, datas=costs)
print("Optimization Finished!")
def test(self):
saver = tf.train.Saver()
saver.restore(self.sess, "network/model/model.ckpt")
# Applying encode and decode over test set
encode_decode = self.sess.run(
self.decoder_op,
feed_dict={self.X: self.dataSet[:self.examples_to_show]})
# Compare original images with their reconstructions
f, a = plt.subplots(2, 10, figsize=(10, 2))
for i in range(self.examples_to_show):
a[2][i].imshow(np.reshape(encode_decode[i], (50, 50, 3)))
a[3][i].imshow(np.reshape(encode_decode[i] - a[i], (50, 50, 3)))
f.show()
plt.draw()
plt.show()
if __name__ == '__main__':
print "reading data..."
dataSet = readCsv("DataSetRawNoDb/data", 1000)
model = AutoEnc_Tensor(dataSet=dataSet)
a = model.train()
# model.test()
INPUT_PHRASE_STATS_FILE = args.INPUT_PHRASE_STATS_FILE
INPUT_CHORDS_FILE = args.INPUT_CHORDS_FILE
BPM = args.BPM
SEEDS = [int(s) for s in args.SEEDS.split(",")]
OUTPUT_FILE = args.OUTPUT_FILE
MS_PER_BEAT = int(round(60.0 / BPM * 1000))
DIVISIONS_PER_BEAT = 4 # e.g. 4 = quarter notes, 8 = eighth notes, etc
BEATS_PER_PHRASE = 16
ROUND_TO_NEAREST = int(round(MS_PER_BEAT / DIVISIONS_PER_BEAT))
VARIANCE_MS = 10 # +/- milliseconds an instrument note should be off by to give it a little more "natural" feel
seedPos = 0
print("Reading data file...")
metaData = readCsv(INPUT_META_FILE)
sampleData = readCsv(INPUT_SAMPLE_FILE)
phraseStats = readCsv(INPUT_PHRASE_STATS_FILE)
chordData = readCsv(INPUT_CHORDS_FILE)
# add note-octave to smapleData
for i, d in enumerate(sampleData):
sampleData[i]["noteOctave"] = "%s%s" % (d["note"], d["octave"])
for i, d in enumerate(chordData):
chordData[i]["noteOctave"] = "%s%s" % (d["note"], d["octave"])
print("Found %s rows in %s" % (len(metaData), INPUT_META_FILE))
print("Found %s rows in %s" % (len(sampleData), INPUT_SAMPLE_FILE))
print("Found %s rows in %s" % (len(phraseStats), INPUT_PHRASE_STATS_FILE))
print("Found %s rows in %s" % (len(chordData), INPUT_CHORDS_FILE))
(88, 164, 172),
(115, 181, 128),
(149, 189, 94),
(185, 189, 74),
(214, 177, 62),
(229, 146, 53),
(229, 94, 43),
(217, 33, 32)
]
random.seed(3)
random.shuffle(colors)
colorCount = len(colors)
# Read files
print("Reading data file...")
data = readCsv(INPUT_FILE)
rowCount = len(data)
print("Found %s rows in %s" % (rowCount, INPUT_FILE))
groups = list(set(d["group"] for d in data))
# Make sure output dirs exist
outDirs = [os.path.dirname(OUTPUT_FILE)]
for outDir in outDirs:
if not os.path.exists(outDir):
os.makedirs(outDir)
im = Image.new(mode="RGB", size=(WIDTH, HEIGHT), color=(0, 0, 0))
draw = ImageDraw.Draw(im)
for i, d in enumerate(data):
import numpy as np
import copy
from matplotlib import pyplot as plt
from utils import readMhd, readCsv, getImgWorldTransfMats, convertToImgCoord, extractCube
from readNoduleList import nodEqDiam
dispFlag = False
# Read nodules csv
csvlines = readCsv('/media/tungthanhlee/SSD/grand_challenge/dataset/LNDB_segmentation/trainset_csv/trainNodules.csv')
# csvlines = readCsv('/home/ad/LungCancer/dataset/trainset_csv/trainNodules.csv')
header = csvlines[0]
nodules = csvlines[1:]
lndloaded = -1
for n in nodules:
vol = float(n[header.index('Volume')])
if nodEqDiam(vol)>3: #only get nodule cubes for nodules>3mm
ctr = np.array([float(n[header.index('x')]), float(n[header.index('y')]), float(n[header.index('z')])])
lnd = int(n[header.index('LNDbID')])
rad = int(n[header.index('RadID')])
# rads = list(map(int,list(n[header.index('RadID')].split(','))))
# radfindings = list(map(int,list(n[header.index('RadFindingID')].split(','))))
finding = int(n[header.index('FindingID')])
# print(lnd,finding,rads,radfindings)
print(lnd,finding,rad,finding)
# Read scan
if lnd!=lndloaded:
def calcNodTexClass(nodtex):
# Compute texture class from texture rating nodtex
texthr = [7 / 3, 11 / 3]
if isinstance(nodtex, float) or isinstance(nodtex, int):
if nodtex >= texthr[0] and nodtex <= texthr[1]:
vclass = 1
elif nodtex > texthr[1]:
vclass = 2
else:
vclass = 0
else: #numpy array
vclass = np.zeros(nodtex.shape)
vclass[np.bitwise_and(nodtex >= texthr[0], nodtex < texthr[1])] = 1
vclass[nodtex >= texthr[1]] = 2
return vclass
if __name__ == "__main__":
# Compute Fleischner score for all trainset nodules
fname_gtNodulesFleischner = '../trainset_csv/trainNodules.csv'
gtNodules = readCsv(fname_gtNodulesFleischner)
gtNodules = joinNodules(gtNodules)
pdFleischner = calcFleischner(gtNodules)
# Compute Fleischner score for all predicted nodules (given volume and texture rating/class/probabilities)
fname_pdNodulesFleischner = 'predictedNodulesC.csv'
pdNodules = readCsv('../submission/' + fname_pdNodulesFleischner)
pdFleischner = calcFleischner(pdNodules)
