def __load_and_store(filename):
patient_data = np.load(filename)['arr_0']
patient_data = patient_data.astype(np.int16)
patient_data = np.delete(patient_data, 2, 0)
X, y, rois, stats = common.load_patient(patient_data,
discard_empty_nodules=True,
output_rois=True,
debug=True,
include_ground_truth=True,
thickness=1,
feature=True)
if not stats:
stats = {'tp': 0, 'fn': 0, 'fp': 0}
logging.info("Patient: %s, stats: %s" %
(filename.split('/')[-1], stats))
if generate_csv:
with open(
os.path.join(OUTPUT_CSV,
'ROIs_dl4_v{}_{}.csv'.format(version, mode)),
'a') as f:
f.write('{},{},{}\n'.format(
filename.split('/')[-1][:-4], stats['tp'],
sum(stats.values())))
# If the patient is empty
return X, y, stats
def __load_and_store(filename):
patient_data = np.load(filename)['arr_0']
patient_data = patient_data.astype(np.int16)
X, y, rois, stats = common.load_patient(patient_data, discard_empty_nodules=True, output_rois=True, debug=True, include_ground_truth=True, thickness=1)
logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats))
return X, y, stats
def __load_and_store(filename):
patient_data = np.load(filename)['arr_0']
ndf = nodules_df[nodules_df['patientid'] == filename.split('/')[-1]]
X, y, rois, stats = common.load_patient(patient_data,
ndf,
output_rois=True,
thickness=1)
logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats))
return X, y, stats
def __load_and_store(filename):
patient_data = np.load(filename)['arr_0']
patid = filename.split('/')[-1]
ndf = nodules_df[nodules_df['patientid'] == patid]
X, y, rois, stats = common.load_patient(patient_data,
ndf,
output_rois=True,
thickness=1)
label = int(label_df[label_df['id'] == patid]['cancer'])
y = [label] * len(X)
logging.info("Patient: %s, cancer:%d, stats: %s" % (patid, label, stats))
return X, y, stats
def __load_and_store(filename):
patient_data = np.load(filename)['arr_0'].astype(np.int16)
ndf = nodules_df[nodules_df['patientid']==filename.split('/')[-1].split('.')[0]]
X, y, rois, stats = common.load_patient(patient_data, ndf, output_rois=True, thickness=1)
if not stats:
stats = {'tp' : 0, 'fn' : 0, 'fp' : 0}
logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats))
if generate_csv:
with open('/home/shared/output/ROIs_dl2_v{}_{}_{}.csv'.format(version_dl2, mode, dataset), 'a') as f:
f.write('{},{},{}\n'.format(filename.split('/')[-1][:-4], stats['tp'], sum(stats.values())))
return X, y, stats
def worker(filename, q, nodules_df=None):
while 1:
if q.qsize() < 10:
patient_data = np.load(filename)['arr_0']
if nodules_df is not None:
ndf = nodules_df[nodules_df['patientid'] == filename.split('/')
[-1]]
X, y, rois, stats = common.load_patient(
patient_data,
ndf,
discard_empty_nodules=False,
output_rois=True,
thickness=1)
else:
X, y, rois, stats = common.load_patient(
patient_data,
discard_empty_nodules=False,
output_rois=True,
thickness=1)
logging.info("Patient: %s, stats: %s" %
(filename.split('/')[-1], stats))
q.put((filename, X, y, rois))
break
f = open(args.output_csv, 'w')
f.write('patientid,nslice,x,y,diameter,score,label\n')
#Set threshold
threshold = -1 #deactivated
nPatients = len(patientFiles)
tStart = time.time()
for pN, patientPath in enumerate(patientFiles):
try:
patientName = patientPath.split('/')[-1][:-4]
patient_data = np.load(patientPath)['arr_0'].astype(np.int16)
logging.info("Patient %s read." % patientName)
ndf = nodules_df[nodules_df['patientid']==patientPath.split('/')[-1].split('.')[0]]
X, y, rois, statsROIGeneration = common.load_patient(patient_data, ndf, discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat)
logging.info("Patient %s ROIS extracted." % patientName)
X = np.asarray(X)
preds = model.predict(X, verbose=2)
logging.info("Predicted patient %s (%d/ %d). Batch results: %d/%d (th=0.7), Max pred %f" % \
( patientPath, pN, nPatients, len([p for p in preds if p>0.7]),len(preds), max(preds)) )
if args.roi_statistics_csv:
logging.info("Patient: %s, stats ROI: %s" % (patientName, statsROIGeneration))
statsROIGeneration['pId'] = patientName
stats_roi_pd = stats_roi_pd.append(statsROIGeneration, ignore_index = True)
for i, p in enumerate(preds):
if p < threshold:
continue
nslice, r = rois[i]
f.write('%s,%d,%d,%d,%.3f,%.5f,%d\n' % (patientName, nslice, r.centroid[0], r.centroid[1], r.equivalent_diameter,preds[i],y[i]))
dataset = args.input_data.split('/')[-2]
f = open(args.output_csv, 'w')
f.write(','.join(csv_headers)+'\n')
#Set threshold
threshold = -1 #deacti ated
nPatients = len(patientFiles)
print('Starting loop over patients...')
for pN, patientPath in enumerate(patientFiles[:10]):
try:
patientName = patientPath.split('/')[-1][:-4]
patient_data = np.load(patientPath)['arr_0'].astype(np.int16)
print('{} - Loading patient {}...'.format(ts(), patientName))
X, y_nodules, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,2]], discard_empty_nodules=False, output_rois=True, thickness=1)
X = np.asarray(X)
preds_nodules = model_dl1.predict(X, verbose=2)
if args.dl4:
X, y_malignancy, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,3]], discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat, feature=True)
y_malignancy = transform_y(y_malignancy)
X = np.asarray(X)
preds_malignancy = model_dl4.predict(X, verbose=2)
if args.dl6:
X, y_lobulation, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,4]], discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat, feature=True)
y_lobulation = transform_y(y_lobulation)
X = np.asarray(X)
preds_lobulation = model_dl6.predict(X, verbose=2)
if args.dl8:
X, y_spiculation, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,5]], discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat, feature=True)
def process_filenames_sequencial(filenames_list,
model_path,
output_csv,
nodules_df=None):
"""Reads regions from queue, predicts nodules and stores in the output file."""
from keras import backend as K
from dl_networks.sample_resnet import ResnetBuilder
from keras.optimizers import Adam
# Model loading inside the listener thread (otherwise keras complains)
K.set_image_dim_ordering('th')
model = ResnetBuilder().build_resnet_50((3, 40, 40), 1)
model.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=['accuracy', 'fmeasure'])
logging.info('Loading existing model %s...' % model_path)
model.load_weights(model_path)
total, errors = 0, 0
f = open(output_csv, 'w')
f.write('patientid,nslice,x,y,diameter,score,label\n')
for filename in filenames_list:
try:
patient_data = np.load(filename)['arr_0']
if nodules_df is not None:
ndf = nodules_df[nodules_df['patientid'] == filename.split('/')
[-1]]
X, y, rois, stats = common.load_patient(
patient_data,
ndf,
discard_empty_nodules=False,
output_rois=True,
thickness=1)
else:
X, y, rois, stats = common.load_patient(
patient_data,
discard_empty_nodules=False,
output_rois=True,
thickness=1)
logging.info("Patient: %s, stats: %s" %
(filename.split('/')[-1], stats))
filename = filename.split('/')[-1]
preds = model.predict(np.asarray(X), verbose=2)
logging.info(
"[Process Sequencial] Predicted patient %d %s. Batch results: %d/%d (th=0.7)"
% (total, filename, len([p for p in preds if p > 0.7
]), len(preds)))
for i in range(len(preds)):
nslice, r = rois[i]
f.write('%s,%d,%d,%d,%.3f,%.5f,%d\n' %
(filename, nslice, r.centroid[0], r.centroid[1],
r.equivalent_diameter, preds[i], y[i]))
total += 1
f.flush()
except Exception as e:
logging.error("[LISTENER] Error processing result, skipping. %s" %
str(e))
errors += 1
logging.info("Stats: %d patients, %d errors" % (total, errors))
f.close()
print('Error. No nodules anotated!')
print(patientName)
no_nodules += 1
continue
extra_slices_df = pd.DataFrame()
for idx, row in ndf.iterrows():
for sl in range(row['nsliceFrom'], row['nsliceTo'] + 1):
aux = row.copy()
aux['nslice'] = sl
extra_slices_df = extra_slices_df.append(aux)
extra_slices_df['nslice'] = extra_slices_df['nslice'].astype(
np.int16)
X, y, rois, stats = common.load_patient(
patient_data,
extra_slices_df,
discard_empty_nodules=True,
output_rois=True,
thickness=1)
else:
X, y, rois, stats = common.load_patient(
patient_data,
output_rois=True,
thickness=1,
feature=True,
nodules_as_rois=1,
discard_empty_nodules=True,
include_ground_truth=True,
ground_truth_only=True)
print(stats)
## True Nodules
