def __load_and_store(filename): patient_data = np.load(filename)['arr_0'] patient_data = patient_data.astype(np.int16) patient_data = np.delete(patient_data, 2, 0) X, y, rois, stats = common.load_patient(patient_data, discard_empty_nodules=True, output_rois=True, debug=True, include_ground_truth=True, thickness=1, feature=True) if not stats: stats = {'tp': 0, 'fn': 0, 'fp': 0} logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats)) if generate_csv: with open( os.path.join(OUTPUT_CSV, 'ROIs_dl4_v{}_{}.csv'.format(version, mode)), 'a') as f: f.write('{},{},{}\n'.format( filename.split('/')[-1][:-4], stats['tp'], sum(stats.values()))) # If the patient is empty return X, y, stats
def __load_and_store(filename): patient_data = np.load(filename)['arr_0'] patient_data = patient_data.astype(np.int16) X, y, rois, stats = common.load_patient(patient_data, discard_empty_nodules=True, output_rois=True, debug=True, include_ground_truth=True, thickness=1) logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats)) return X, y, stats
def __load_and_store(filename): patient_data = np.load(filename)['arr_0'] ndf = nodules_df[nodules_df['patientid'] == filename.split('/')[-1]] X, y, rois, stats = common.load_patient(patient_data, ndf, output_rois=True, thickness=1) logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats)) return X, y, stats
def __load_and_store(filename): patient_data = np.load(filename)['arr_0'] patid = filename.split('/')[-1] ndf = nodules_df[nodules_df['patientid'] == patid] X, y, rois, stats = common.load_patient(patient_data, ndf, output_rois=True, thickness=1) label = int(label_df[label_df['id'] == patid]['cancer']) y = [label] * len(X) logging.info("Patient: %s, cancer:%d, stats: %s" % (patid, label, stats)) return X, y, stats
def __load_and_store(filename): patient_data = np.load(filename)['arr_0'].astype(np.int16) ndf = nodules_df[nodules_df['patientid']==filename.split('/')[-1].split('.')[0]] X, y, rois, stats = common.load_patient(patient_data, ndf, output_rois=True, thickness=1) if not stats: stats = {'tp' : 0, 'fn' : 0, 'fp' : 0} logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats)) if generate_csv: with open('/home/shared/output/ROIs_dl2_v{}_{}_{}.csv'.format(version_dl2, mode, dataset), 'a') as f: f.write('{},{},{}\n'.format(filename.split('/')[-1][:-4], stats['tp'], sum(stats.values()))) return X, y, stats
def worker(filename, q, nodules_df=None): while 1: if q.qsize() < 10: patient_data = np.load(filename)['arr_0'] if nodules_df is not None: ndf = nodules_df[nodules_df['patientid'] == filename.split('/') [-1]] X, y, rois, stats = common.load_patient( patient_data, ndf, discard_empty_nodules=False, output_rois=True, thickness=1) else: X, y, rois, stats = common.load_patient( patient_data, discard_empty_nodules=False, output_rois=True, thickness=1) logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats)) q.put((filename, X, y, rois)) break
f = open(args.output_csv, 'w') f.write('patientid,nslice,x,y,diameter,score,label\n') #Set threshold threshold = -1 #deactivated nPatients = len(patientFiles) tStart = time.time() for pN, patientPath in enumerate(patientFiles): try: patientName = patientPath.split('/')[-1][:-4] patient_data = np.load(patientPath)['arr_0'].astype(np.int16) logging.info("Patient %s read." % patientName) ndf = nodules_df[nodules_df['patientid']==patientPath.split('/')[-1].split('.')[0]] X, y, rois, statsROIGeneration = common.load_patient(patient_data, ndf, discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat) logging.info("Patient %s ROIS extracted." % patientName) X = np.asarray(X) preds = model.predict(X, verbose=2) logging.info("Predicted patient %s (%d/ %d). Batch results: %d/%d (th=0.7), Max pred %f" % \ ( patientPath, pN, nPatients, len([p for p in preds if p>0.7]),len(preds), max(preds)) ) if args.roi_statistics_csv: logging.info("Patient: %s, stats ROI: %s" % (patientName, statsROIGeneration)) statsROIGeneration['pId'] = patientName stats_roi_pd = stats_roi_pd.append(statsROIGeneration, ignore_index = True) for i, p in enumerate(preds): if p < threshold: continue nslice, r = rois[i] f.write('%s,%d,%d,%d,%.3f,%.5f,%d\n' % (patientName, nslice, r.centroid[0], r.centroid[1], r.equivalent_diameter,preds[i],y[i]))
dataset = args.input_data.split('/')[-2] f = open(args.output_csv, 'w') f.write(','.join(csv_headers)+'\n') #Set threshold threshold = -1 #deacti ated nPatients = len(patientFiles) print('Starting loop over patients...') for pN, patientPath in enumerate(patientFiles[:10]): try: patientName = patientPath.split('/')[-1][:-4] patient_data = np.load(patientPath)['arr_0'].astype(np.int16) print('{} - Loading patient {}...'.format(ts(), patientName)) X, y_nodules, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,2]], discard_empty_nodules=False, output_rois=True, thickness=1) X = np.asarray(X) preds_nodules = model_dl1.predict(X, verbose=2) if args.dl4: X, y_malignancy, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,3]], discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat, feature=True) y_malignancy = transform_y(y_malignancy) X = np.asarray(X) preds_malignancy = model_dl4.predict(X, verbose=2) if args.dl6: X, y_lobulation, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,4]], discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat, feature=True) y_lobulation = transform_y(y_lobulation) X = np.asarray(X) preds_lobulation = model_dl6.predict(X, verbose=2) if args.dl8: X, y_spiculation, rois, statsROIGeneration = common.load_patient(patient_data[[0,1,5]], discard_empty_nodules=False, output_rois=True, thickness=1, convertToFloat = args.convertToFloat, feature=True)
def process_filenames_sequencial(filenames_list, model_path, output_csv, nodules_df=None): """Reads regions from queue, predicts nodules and stores in the output file.""" from keras import backend as K from dl_networks.sample_resnet import ResnetBuilder from keras.optimizers import Adam # Model loading inside the listener thread (otherwise keras complains) K.set_image_dim_ordering('th') model = ResnetBuilder().build_resnet_50((3, 40, 40), 1) model.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy', metrics=['accuracy', 'fmeasure']) logging.info('Loading existing model %s...' % model_path) model.load_weights(model_path) total, errors = 0, 0 f = open(output_csv, 'w') f.write('patientid,nslice,x,y,diameter,score,label\n') for filename in filenames_list: try: patient_data = np.load(filename)['arr_0'] if nodules_df is not None: ndf = nodules_df[nodules_df['patientid'] == filename.split('/') [-1]] X, y, rois, stats = common.load_patient( patient_data, ndf, discard_empty_nodules=False, output_rois=True, thickness=1) else: X, y, rois, stats = common.load_patient( patient_data, discard_empty_nodules=False, output_rois=True, thickness=1) logging.info("Patient: %s, stats: %s" % (filename.split('/')[-1], stats)) filename = filename.split('/')[-1] preds = model.predict(np.asarray(X), verbose=2) logging.info( "[Process Sequencial] Predicted patient %d %s. Batch results: %d/%d (th=0.7)" % (total, filename, len([p for p in preds if p > 0.7 ]), len(preds))) for i in range(len(preds)): nslice, r = rois[i] f.write('%s,%d,%d,%d,%.3f,%.5f,%d\n' % (filename, nslice, r.centroid[0], r.centroid[1], r.equivalent_diameter, preds[i], y[i])) total += 1 f.flush() except Exception as e: logging.error("[LISTENER] Error processing result, skipping. %s" % str(e)) errors += 1 logging.info("Stats: %d patients, %d errors" % (total, errors)) f.close()
print('Error. No nodules anotated!') print(patientName) no_nodules += 1 continue extra_slices_df = pd.DataFrame() for idx, row in ndf.iterrows(): for sl in range(row['nsliceFrom'], row['nsliceTo'] + 1): aux = row.copy() aux['nslice'] = sl extra_slices_df = extra_slices_df.append(aux) extra_slices_df['nslice'] = extra_slices_df['nslice'].astype( np.int16) X, y, rois, stats = common.load_patient( patient_data, extra_slices_df, discard_empty_nodules=True, output_rois=True, thickness=1) else: X, y, rois, stats = common.load_patient( patient_data, output_rois=True, thickness=1, feature=True, nodules_as_rois=1, discard_empty_nodules=True, include_ground_truth=True, ground_truth_only=True) print(stats) ## True Nodules