def run_collecting_neuron_distance(): output_csv=data_DIR+"/analysis_results/neuron_distance_strict.csv" #collect results from log files df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance','neuron_distance_diff','neuron_distance_perc')) count =0 for recon_dir in subdirs[1:]: folder_name = recon_dir.split('/')[-1] if 'processed' in folder_name: print recon_dir files = glob.glob(recon_dir+'/../00_*strict.swc') if len(files)>0: gs_swc_file =files[0] image_folder_name = recon_dir.split('/')[-2] image_id = int(image_folder_name.split(".")[0]) image_file_name = df_lookup_table.image_file_name[image_id-1] # auto recons swc_files = glob.glob(recon_dir+'/*.strict.swc') for i in range(len(swc_files)) : input = swc_files[i] print input out_fn=recon_dir+"/../processed/"+input.split('/')[-1]+'.dist.r.log' algorithm = rp.matchFileToAlgorithmName( input.split('/')[-1]) if path.isfile(out_fn): nd = bn.read_neuron_dist_log(out_fn) df_neuron_distance.loc[count] = [image_file_name,input, gs_swc_file, algorithm, nd['ave'],nd['diff'],nd['perc']] else: print "Warning: no neuron distance log output for "+out_fn +" :output NAs." df_neuron_distance.loc[count] = [image_file_name,input, gs_swc_file, algorithm, np.nan, np.nan, np.nan] count= count+1 #consensus consensus_swc= recon_dir+'/../consensus3.strict.swc' consensus_swc_log = recon_dir+'/../consensus3.strict.swc.dist.r.log' if path.exists(consensus_swc_log): nd = bn.read_neuron_dist_log(consensus_swc_log) df_neuron_distance.loc[count] = [image_file_name,consensus_swc, gs_swc_file, algorithm, nd['ave'],nd['diff'],nd['perc']] else: print "Warning: no neuron distance log output for "+consensus_swc +" :output NAs." df_neuron_distance.loc[count] = [image_file_name,consensus_swc, gs_swc_file, algorithm, np.nan, np.nan, np.nan] count = count+1 df_neuron_distance['neuron_difference'] = df_neuron_distance['neuron_distance_diff'] *df_neuron_distance['neuron_distance_perc'] df_neuron_distance.to_csv(output_csv,index=False) print "output:"+output_csv
def collect_distances(input_dir, output_neuron_distance_csv): #collect results from log files df_neuron_distance = pd.DataFrame( columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance_12', 'neuron_distance_21', 'neuron_distance_ave', 'neuron_distance_diff', 'neuron_distance_perc')) i = 0 for dirpath, dnames, fnames in os.walk(input_dir): for f in fnames: if f.endswith(".r.log"): log_f = (os.path.join(dirpath, f)) image = log_f.split('/')[-2] algorithm = rp.matchFileToAlgorithmName(log_f[0:-6]) nd = bn.read_neuron_dist_log(log_f) df_neuron_distance.loc[i] = [ image, nd['input_file1'], nd['input_file2'], algorithm, nd['dist_12'], nd['dist_21'], nd['ave'], nd['diff'], nd['perc'] ] i = i + 1 df_neuron_distance = df_neuron_distance[ df_neuron_distance['neuron_distance_ave'] != -1] # empty no df_neuron_distance.to_csv(output_neuron_distance_csv, index=False) return
def collect_distances(input_dir, output_neuron_distance_csv): #collect results from log files df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance_12','neuron_distance_21', 'neuron_distance_ave','neuron_distance_diff', 'neuron_distance_perc')) i=0 for dirpath, dnames, fnames in os.walk(input_dir): for f in fnames: if f.endswith(".r.log"): log_f= (os.path.join(dirpath, f)) image = log_f.split('/')[-2] algorithm = rp.matchFileToAlgorithmName(log_f[0:-6]) nd = bn.read_neuron_dist_log(log_f) df_neuron_distance.loc[i] = [image,nd['input_file1'], nd['input_file2'], algorithm, nd['dist_12'],nd['dist_21'],nd['ave'],nd['diff'], nd['perc']] i=i+1 df_neuron_distance = df_neuron_distance[df_neuron_distance['neuron_distance_ave'] != -1] # empty no df_neuron_distance.to_csv(output_neuron_distance_csv, index=False) return
def cal_neuron_dist(input_csv_file, output_csv, overwrite_existing=1, GEN_QSUB=0): df_input = pd.read_csv(input_csv_file) df_already_have = pd.DataFrame(columns=df_input.columns) # if (not overwrite_existing): # if os.path.isfile(old_output_csv): # # only run new data # df_old = pd.read_csv(old_output_csv) # df_already_have = pd.merge(df_input, df_old, on='swc_file') # print "there are already "+ str(df_already_have['swc_file'].size) +" swcs calculated" output_dir = os.path.dirname(output_csv) print " Calculate neuron distances:" for i in range(df_input.image_file_name.size): print "swc file :" + str(i) swc_f = df_input.iloc[i].swc_file log_file = df_input.iloc[i].swc_file + ".r.log" #if not swc_f in list(df_already_have['swc_file']) if overwrite_existing or not os.path.isfile(log_file): bn.run_neuron_dist(swc_f, df_input.iloc[i].gold_swc_file, log_file, GEN_QSUB, output_dir + "/nd") #collect results from log files df_neuron_distance = pd.DataFrame( columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance_12', 'neuron_distance_21', 'neuron_distance_ave', 'neuron_distance_diff', 'neuron_distance_perc')) for i in range(df_input.image_file_name.size): tmp = df_input.iloc[i].swc_file logfile = tmp + ".r.log" if path.isfile(logfile): nd = bn.read_neuron_dist_log(logfile) df_neuron_distance.loc[i] = [ df_input.iloc[i].image_file_name, df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, nd['dist_12'], nd['dist_21'], nd['ave'], nd['diff'], nd['perc'] ] else: print "Warning: no neuron distance log output for " + tmp + " :output NAs." df_neuron_distance.loc[i] = [ df_input.iloc[i].image_file_name, df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, np.nan, np.nan, np.nan, np.nan, np.nan ] df_neuron_distance['neuron_difference'] = df_neuron_distance[ 'neuron_distance_diff'] * df_neuron_distance['neuron_distance_perc'] df_neuron_distance.to_csv(output_csv, index=False) print "output:" + output_csv return
def cal_neuron_dist(input_csv_file,output_csv,overwrite_existing = 1,GEN_QSUB = 0 ): df_input = pd.read_csv(input_csv_file) df_already_have = pd.DataFrame(columns = df_input.columns) # if (not overwrite_existing): # if os.path.isfile(old_output_csv): # # only run new data # df_old = pd.read_csv(old_output_csv) # df_already_have = pd.merge(df_input, df_old, on='swc_file') # print "there are already "+ str(df_already_have['swc_file'].size) +" swcs calculated" output_dir = os.path.dirname(output_csv) print " Calculate neuron distances:" for i in range(df_input.image_file_name.size): print "swc file :" + str(i) swc_f = df_input.iloc[i].swc_file log_file = df_input.iloc[i].swc_file + ".r.log" #if not swc_f in list(df_already_have['swc_file']) if overwrite_existing or not os.path.isfile(log_file) : bn.run_neuron_dist(swc_f, df_input.iloc[i].gold_swc_file,log_file, GEN_QSUB, output_dir+"/nd") #collect results from log files df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance_12','neuron_distance_21', 'neuron_distance_ave','neuron_distance_diff', 'neuron_distance_perc')) for i in range(df_input.image_file_name.size): tmp = df_input.iloc[i].swc_file logfile = tmp + ".r.log" if path.isfile(logfile): nd = bn.read_neuron_dist_log(logfile) df_neuron_distance.loc[i] = [df_input.iloc[i].image_file_name,df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, nd['dist_12'],nd['dist_21'],nd['ave'],nd['diff'], nd['perc']] else: print "Warning: no neuron distance log output for "+tmp +" :output NAs." df_neuron_distance.loc[i]= [df_input.iloc[i].image_file_name,df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, np.nan, np.nan, np.nan, np.nan, np.nan] df_neuron_distance['neuron_difference'] = df_neuron_distance['neuron_distance_diff'] *df_neuron_distance['neuron_distance_perc'] df_neuron_distance.to_csv(output_csv,index=False) print "output:"+output_csv return
def collect_neuron_distance(input_distance_log_file_list, output_distance_csv, lookup_image_id_table_file): df_input = pd.read_csv( input_distance_log_file_list, header=None) # txt file contains the list of log files df_lookup_table = pd.read_csv(lookup_image_id_table_file) df_neuron_distance = pd.DataFrame( columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance_12', 'neuron_distance_21', 'neuron_distance_ave', 'neuron_distance_diff', 'neuron_distance_perc')) for i in range(df_input.size): logfile_path = df_input.iloc[i][0] image_id = int(logfile_path.split("/")[-3]) if image_id > df_lookup_table.image_file_name.size: print "error in looking image ids" image_file_name = df_lookup_table.image_file_name[image_id - 1] if path.isfile(logfile_path): nd = bn.read_neuron_dist_log(logfile_path) algorithm = matchFileToAlgorithmName(logfile_path.split('/')[-1]) ##return {'input_file1':input_file1, 'input_file2':input_file2,'dist_12':d_12,'dist_21':d_21, 'ave': ave, 'diff': diff, 'perc': perc} df_neuron_distance.loc[i] = [ image_file_name, nd['input_file1'], nd['input_file2'], algorithm, nd['dist_12'], nd['dist_21'], nd['ave'], nd['diff'], nd['perc'] ] else: print "Warning: no neuron distance log output for " + image_file_name + " :output NAs." df_neuron_distance.loc[i] = [ image_file_name, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan ] df_neuron_distance['neuron_difference'] = df_neuron_distance[ 'neuron_distance_diff'] * df_neuron_distance['neuron_distance_perc'] df_neuron_distance.to_csv(output_distance_csv, index=False) print "output:" + output_distance_csv return
def collect_neuron_distance(input_distance_log_file_list, output_distance_csv,lookup_image_id_table_file): df_input = pd.read_csv(input_distance_log_file_list, header=None) # txt file contains the list of log files df_lookup_table = pd.read_csv(lookup_image_id_table_file) df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance_12','neuron_distance_21', 'neuron_distance_ave','neuron_distance_diff', 'neuron_distance_perc')) for i in range(df_input.size): logfile_path = df_input.iloc[i][0] image_id = int(logfile_path.split("/")[-3]) if image_id > df_lookup_table.image_file_name.size: print "error in looking image ids" image_file_name = df_lookup_table.image_file_name[image_id-1] if path.isfile(logfile_path): nd = bn.read_neuron_dist_log(logfile_path) algorithm = matchFileToAlgorithmName( logfile_path.split('/')[-1]) ##return {'input_file1':input_file1, 'input_file2':input_file2,'dist_12':d_12,'dist_21':d_21, 'ave': ave, 'diff': diff, 'perc': perc} df_neuron_distance.loc[i] = [image_file_name,nd['input_file1'], nd['input_file2'], algorithm, nd['dist_12'],nd['dist_21'],nd['ave'],nd['diff'], nd['perc'] ] else: print "Warning: no neuron distance log output for "+image_file_name +" :output NAs." df_neuron_distance.loc[i]= [image_file_name ,np.nan,np.nan, np.nan, np.nan,np.nan,np.nan, np.nan, np.nan] df_neuron_distance['neuron_difference'] = df_neuron_distance['neuron_distance_diff'] *df_neuron_distance['neuron_distance_perc'] df_neuron_distance.to_csv(output_distance_csv,index=False) print "output:"+output_distance_csv return
def run_collecting_neuron_distance(): output_csv = data_DIR + "/analysis_results/neuron_distance_strict.csv" #collect results from log files df_neuron_distance = pd.DataFrame(columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance', 'neuron_distance_diff', 'neuron_distance_perc')) count = 0 for recon_dir in subdirs[1:]: folder_name = recon_dir.split('/')[-1] if 'processed' in folder_name: print recon_dir files = glob.glob(recon_dir + '/../00_*strict.swc') if len(files) > 0: gs_swc_file = files[0] image_folder_name = recon_dir.split('/')[-2] image_id = int(image_folder_name.split(".")[0]) image_file_name = df_lookup_table.image_file_name[image_id - 1] # auto recons swc_files = glob.glob(recon_dir + '/*.strict.swc') for i in range(len(swc_files)): input = swc_files[i] print input out_fn = recon_dir + "/../processed/" + input.split( '/')[-1] + '.dist.r.log' algorithm = rp.matchFileToAlgorithmName(input.split('/')[-1]) if path.isfile(out_fn): nd = bn.read_neuron_dist_log(out_fn) df_neuron_distance.loc[count] = [ image_file_name, input, gs_swc_file, algorithm, nd['ave'], nd['diff'], nd['perc'] ] else: print "Warning: no neuron distance log output for " + out_fn + " :output NAs." df_neuron_distance.loc[count] = [ image_file_name, input, gs_swc_file, algorithm, np.nan, np.nan, np.nan ] count = count + 1 #consensus consensus_swc = recon_dir + '/../consensus3.strict.swc' consensus_swc_log = recon_dir + '/../consensus3.strict.swc.dist.r.log' if path.exists(consensus_swc_log): nd = bn.read_neuron_dist_log(consensus_swc_log) df_neuron_distance.loc[count] = [ image_file_name, consensus_swc, gs_swc_file, algorithm, nd['ave'], nd['diff'], nd['perc'] ] else: print "Warning: no neuron distance log output for " + consensus_swc + " :output NAs." df_neuron_distance.loc[count] = [ image_file_name, consensus_swc, gs_swc_file, algorithm, np.nan, np.nan, np.nan ] count = count + 1 df_neuron_distance['neuron_difference'] = df_neuron_distance[ 'neuron_distance_diff'] * df_neuron_distance['neuron_distance_perc'] df_neuron_distance.to_csv(output_csv, index=False) print "output:" + output_csv
############################################### COLLECT_NEURON_DISTANCE = 1 if COLLECT_NEURON_DISTANCE: df_neuron_distance = pd.DataFrame(columns=('swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance')) idx = 0 for i in range(final_images.size): imageName = final_images[i] df_image = df_results[df_results.image == imageName] df_gold_image = df_gold[df_gold.image == imageName].iloc[0] for j in range(df_image.shape[0]): df_swc = df_image.iloc[j] logfile = df_swc.swc_file + ".r.log" if path.isfile(logfile): nd = bn.read_neuron_dist_log(logfile) algorithm = df_swc.algorithm df_neuron_distance.loc[idx] = [df_swc.swc_file, df_gold_image.swc_file, algorithm, nd['ave']] idx = idx + 1 df_neuron_distance.to_csv(data_DIR + "/neuron_distance.r.csv") ###### plot################################################## df_nd = pd.read_csv(data_DIR + "/neuron_distance.r.csv") outputDir = data_DIR + "/neuron_dist_plots_r" if not path.exists(outputDir): os.mkdir(outputDir) CASE_BY_CASE_PLOT = 1 if CASE_BY_CASE_PLOT: images = np.unique(df_nd.gold_swc_file)