def run_collecting_neuron_distance():
output_csv=data_DIR+"/analysis_results/neuron_distance_strict.csv"
#collect results from log files
df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance','neuron_distance_diff','neuron_distance_perc'))
count =0
for recon_dir in subdirs[1:]:
folder_name = recon_dir.split('/')[-1]
if 'processed' in folder_name:
print recon_dir
files = glob.glob(recon_dir+'/../00_*strict.swc')
if len(files)>0:
gs_swc_file =files[0]
image_folder_name = recon_dir.split('/')[-2]
image_id = int(image_folder_name.split(".")[0])
image_file_name = df_lookup_table.image_file_name[image_id-1]
# auto recons
swc_files = glob.glob(recon_dir+'/*.strict.swc')
for i in range(len(swc_files)) :
input = swc_files[i]
print input
out_fn=recon_dir+"/../processed/"+input.split('/')[-1]+'.dist.r.log'
algorithm = rp.matchFileToAlgorithmName( input.split('/')[-1])
if path.isfile(out_fn):
nd = bn.read_neuron_dist_log(out_fn)
df_neuron_distance.loc[count] = [image_file_name,input, gs_swc_file, algorithm, nd['ave'],nd['diff'],nd['perc']]
else:
print "Warning: no neuron distance log output for "+out_fn +" :output NAs."
df_neuron_distance.loc[count] = [image_file_name,input, gs_swc_file, algorithm, np.nan, np.nan, np.nan]
count= count+1
#consensus
consensus_swc= recon_dir+'/../consensus3.strict.swc'
consensus_swc_log = recon_dir+'/../consensus3.strict.swc.dist.r.log'
if path.exists(consensus_swc_log):
nd = bn.read_neuron_dist_log(consensus_swc_log)
df_neuron_distance.loc[count] = [image_file_name,consensus_swc, gs_swc_file, algorithm, nd['ave'],nd['diff'],nd['perc']]
else:
print "Warning: no neuron distance log output for "+consensus_swc +" :output NAs."
df_neuron_distance.loc[count] = [image_file_name,consensus_swc, gs_swc_file, algorithm, np.nan, np.nan, np.nan]
count = count+1
df_neuron_distance['neuron_difference'] = df_neuron_distance['neuron_distance_diff'] *df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_csv,index=False)
print "output:"+output_csv
def collect_distances(input_dir, output_neuron_distance_csv):
#collect results from log files
df_neuron_distance = pd.DataFrame(
columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm',
'neuron_distance_12', 'neuron_distance_21',
'neuron_distance_ave', 'neuron_distance_diff',
'neuron_distance_perc'))
i = 0
for dirpath, dnames, fnames in os.walk(input_dir):
for f in fnames:
if f.endswith(".r.log"):
log_f = (os.path.join(dirpath, f))
image = log_f.split('/')[-2]
algorithm = rp.matchFileToAlgorithmName(log_f[0:-6])
nd = bn.read_neuron_dist_log(log_f)
df_neuron_distance.loc[i] = [
image, nd['input_file1'], nd['input_file2'], algorithm,
nd['dist_12'], nd['dist_21'], nd['ave'], nd['diff'],
nd['perc']
]
i = i + 1
df_neuron_distance = df_neuron_distance[
df_neuron_distance['neuron_distance_ave'] != -1] # empty no
df_neuron_distance.to_csv(output_neuron_distance_csv, index=False)
return
def collect_distances(input_dir, output_neuron_distance_csv):
#collect results from log files
df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file',
'algorithm',
'neuron_distance_12','neuron_distance_21',
'neuron_distance_ave','neuron_distance_diff',
'neuron_distance_perc'))
i=0
for dirpath, dnames, fnames in os.walk(input_dir):
for f in fnames:
if f.endswith(".r.log"):
log_f= (os.path.join(dirpath, f))
image = log_f.split('/')[-2]
algorithm = rp.matchFileToAlgorithmName(log_f[0:-6])
nd = bn.read_neuron_dist_log(log_f)
df_neuron_distance.loc[i] = [image,nd['input_file1'], nd['input_file2'], algorithm, nd['dist_12'],nd['dist_21'],nd['ave'],nd['diff'],
nd['perc']]
i=i+1
df_neuron_distance = df_neuron_distance[df_neuron_distance['neuron_distance_ave'] != -1] # empty no
df_neuron_distance.to_csv(output_neuron_distance_csv, index=False)
return
def cal_neuron_dist(input_csv_file,
output_csv,
overwrite_existing=1,
GEN_QSUB=0):
df_input = pd.read_csv(input_csv_file)
df_already_have = pd.DataFrame(columns=df_input.columns)
# if (not overwrite_existing):
# if os.path.isfile(old_output_csv):
# # only run new data
# df_old = pd.read_csv(old_output_csv)
# df_already_have = pd.merge(df_input, df_old, on='swc_file')
# print "there are already "+ str(df_already_have['swc_file'].size) +" swcs calculated"
output_dir = os.path.dirname(output_csv)
print " Calculate neuron distances:"
for i in range(df_input.image_file_name.size):
print "swc file :" + str(i)
swc_f = df_input.iloc[i].swc_file
log_file = df_input.iloc[i].swc_file + ".r.log"
#if not swc_f in list(df_already_have['swc_file'])
if overwrite_existing or not os.path.isfile(log_file):
bn.run_neuron_dist(swc_f, df_input.iloc[i].gold_swc_file, log_file,
GEN_QSUB, output_dir + "/nd")
#collect results from log files
df_neuron_distance = pd.DataFrame(
columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm',
'neuron_distance_12', 'neuron_distance_21',
'neuron_distance_ave', 'neuron_distance_diff',
'neuron_distance_perc'))
for i in range(df_input.image_file_name.size):
tmp = df_input.iloc[i].swc_file
logfile = tmp + ".r.log"
if path.isfile(logfile):
nd = bn.read_neuron_dist_log(logfile)
df_neuron_distance.loc[i] = [
df_input.iloc[i].image_file_name, df_input.iloc[i].swc_file,
df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm,
nd['dist_12'], nd['dist_21'], nd['ave'], nd['diff'], nd['perc']
]
else:
print "Warning: no neuron distance log output for " + tmp + " :output NAs."
df_neuron_distance.loc[i] = [
df_input.iloc[i].image_file_name, df_input.iloc[i].swc_file,
df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm,
np.nan, np.nan, np.nan, np.nan, np.nan
]
df_neuron_distance['neuron_difference'] = df_neuron_distance[
'neuron_distance_diff'] * df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_csv, index=False)
print "output:" + output_csv
return
def cal_neuron_dist(input_csv_file,output_csv,overwrite_existing = 1,GEN_QSUB = 0 ):
df_input = pd.read_csv(input_csv_file)
df_already_have = pd.DataFrame(columns = df_input.columns)
# if (not overwrite_existing):
# if os.path.isfile(old_output_csv):
# # only run new data
# df_old = pd.read_csv(old_output_csv)
# df_already_have = pd.merge(df_input, df_old, on='swc_file')
# print "there are already "+ str(df_already_have['swc_file'].size) +" swcs calculated"
output_dir = os.path.dirname(output_csv)
print " Calculate neuron distances:"
for i in range(df_input.image_file_name.size):
print "swc file :" + str(i)
swc_f = df_input.iloc[i].swc_file
log_file = df_input.iloc[i].swc_file + ".r.log"
#if not swc_f in list(df_already_have['swc_file'])
if overwrite_existing or not os.path.isfile(log_file) :
bn.run_neuron_dist(swc_f, df_input.iloc[i].gold_swc_file,log_file, GEN_QSUB, output_dir+"/nd")
#collect results from log files
df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file',
'algorithm',
'neuron_distance_12','neuron_distance_21',
'neuron_distance_ave','neuron_distance_diff',
'neuron_distance_perc'))
for i in range(df_input.image_file_name.size):
tmp = df_input.iloc[i].swc_file
logfile = tmp + ".r.log"
if path.isfile(logfile):
nd = bn.read_neuron_dist_log(logfile)
df_neuron_distance.loc[i] = [df_input.iloc[i].image_file_name,df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, nd['dist_12'],nd['dist_21'],nd['ave'],nd['diff'],
nd['perc']]
else:
print "Warning: no neuron distance log output for "+tmp +" :output NAs."
df_neuron_distance.loc[i]= [df_input.iloc[i].image_file_name,df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, np.nan, np.nan, np.nan, np.nan, np.nan]
df_neuron_distance['neuron_difference'] = df_neuron_distance['neuron_distance_diff'] *df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_csv,index=False)
print "output:"+output_csv
return
def collect_neuron_distance(input_distance_log_file_list, output_distance_csv,
lookup_image_id_table_file):
df_input = pd.read_csv(
input_distance_log_file_list,
header=None) # txt file contains the list of log files
df_lookup_table = pd.read_csv(lookup_image_id_table_file)
df_neuron_distance = pd.DataFrame(
columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm',
'neuron_distance_12', 'neuron_distance_21',
'neuron_distance_ave', 'neuron_distance_diff',
'neuron_distance_perc'))
for i in range(df_input.size):
logfile_path = df_input.iloc[i][0]
image_id = int(logfile_path.split("/")[-3])
if image_id > df_lookup_table.image_file_name.size:
print "error in looking image ids"
image_file_name = df_lookup_table.image_file_name[image_id - 1]
if path.isfile(logfile_path):
nd = bn.read_neuron_dist_log(logfile_path)
algorithm = matchFileToAlgorithmName(logfile_path.split('/')[-1])
##return {'input_file1':input_file1, 'input_file2':input_file2,'dist_12':d_12,'dist_21':d_21, 'ave': ave, 'diff': diff, 'perc': perc}
df_neuron_distance.loc[i] = [
image_file_name, nd['input_file1'], nd['input_file2'],
algorithm, nd['dist_12'], nd['dist_21'], nd['ave'], nd['diff'],
nd['perc']
]
else:
print "Warning: no neuron distance log output for " + image_file_name + " :output NAs."
df_neuron_distance.loc[i] = [
image_file_name, np.nan, np.nan, np.nan, np.nan, np.nan,
np.nan, np.nan, np.nan
]
df_neuron_distance['neuron_difference'] = df_neuron_distance[
'neuron_distance_diff'] * df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_distance_csv, index=False)
print "output:" + output_distance_csv
return
def collect_neuron_distance(input_distance_log_file_list, output_distance_csv,lookup_image_id_table_file):
df_input = pd.read_csv(input_distance_log_file_list, header=None) # txt file contains the list of log files
df_lookup_table = pd.read_csv(lookup_image_id_table_file)
df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file',
'algorithm',
'neuron_distance_12','neuron_distance_21',
'neuron_distance_ave','neuron_distance_diff',
'neuron_distance_perc'))
for i in range(df_input.size):
logfile_path = df_input.iloc[i][0]
image_id = int(logfile_path.split("/")[-3])
if image_id > df_lookup_table.image_file_name.size:
print "error in looking image ids"
image_file_name = df_lookup_table.image_file_name[image_id-1]
if path.isfile(logfile_path):
nd = bn.read_neuron_dist_log(logfile_path)
algorithm = matchFileToAlgorithmName( logfile_path.split('/')[-1])
##return {'input_file1':input_file1, 'input_file2':input_file2,'dist_12':d_12,'dist_21':d_21, 'ave': ave, 'diff': diff, 'perc': perc}
df_neuron_distance.loc[i] = [image_file_name,nd['input_file1'], nd['input_file2'],
algorithm, nd['dist_12'],nd['dist_21'],nd['ave'],nd['diff'],
nd['perc'] ]
else:
print "Warning: no neuron distance log output for "+image_file_name +" :output NAs."
df_neuron_distance.loc[i]= [image_file_name ,np.nan,np.nan, np.nan, np.nan,np.nan,np.nan, np.nan, np.nan]
df_neuron_distance['neuron_difference'] = df_neuron_distance['neuron_distance_diff'] *df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_distance_csv,index=False)
print "output:"+output_distance_csv
return
def run_collecting_neuron_distance():
output_csv = data_DIR + "/analysis_results/neuron_distance_strict.csv"
#collect results from log files
df_neuron_distance = pd.DataFrame(columns=('image_file_name', 'swc_file',
'gold_swc_file', 'algorithm',
'neuron_distance',
'neuron_distance_diff',
'neuron_distance_perc'))
count = 0
for recon_dir in subdirs[1:]:
folder_name = recon_dir.split('/')[-1]
if 'processed' in folder_name:
print recon_dir
files = glob.glob(recon_dir + '/../00_*strict.swc')
if len(files) > 0:
gs_swc_file = files[0]
image_folder_name = recon_dir.split('/')[-2]
image_id = int(image_folder_name.split(".")[0])
image_file_name = df_lookup_table.image_file_name[image_id - 1]
# auto recons
swc_files = glob.glob(recon_dir + '/*.strict.swc')
for i in range(len(swc_files)):
input = swc_files[i]
print input
out_fn = recon_dir + "/../processed/" + input.split(
'/')[-1] + '.dist.r.log'
algorithm = rp.matchFileToAlgorithmName(input.split('/')[-1])
if path.isfile(out_fn):
nd = bn.read_neuron_dist_log(out_fn)
df_neuron_distance.loc[count] = [
image_file_name, input, gs_swc_file, algorithm,
nd['ave'], nd['diff'], nd['perc']
]
else:
print "Warning: no neuron distance log output for " + out_fn + " :output NAs."
df_neuron_distance.loc[count] = [
image_file_name, input, gs_swc_file, algorithm, np.nan,
np.nan, np.nan
]
count = count + 1
#consensus
consensus_swc = recon_dir + '/../consensus3.strict.swc'
consensus_swc_log = recon_dir + '/../consensus3.strict.swc.dist.r.log'
if path.exists(consensus_swc_log):
nd = bn.read_neuron_dist_log(consensus_swc_log)
df_neuron_distance.loc[count] = [
image_file_name, consensus_swc, gs_swc_file, algorithm,
nd['ave'], nd['diff'], nd['perc']
]
else:
print "Warning: no neuron distance log output for " + consensus_swc + " :output NAs."
df_neuron_distance.loc[count] = [
image_file_name, consensus_swc, gs_swc_file, algorithm,
np.nan, np.nan, np.nan
]
count = count + 1
df_neuron_distance['neuron_difference'] = df_neuron_distance[
'neuron_distance_diff'] * df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_csv, index=False)
print "output:" + output_csv
###############################################
COLLECT_NEURON_DISTANCE = 1
if COLLECT_NEURON_DISTANCE:
df_neuron_distance = pd.DataFrame(columns=('swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance'))
idx = 0
for i in range(final_images.size):
imageName = final_images[i]
df_image = df_results[df_results.image == imageName]
df_gold_image = df_gold[df_gold.image == imageName].iloc[0]
for j in range(df_image.shape[0]):
df_swc = df_image.iloc[j]
logfile = df_swc.swc_file + ".r.log"
if path.isfile(logfile):
nd = bn.read_neuron_dist_log(logfile)
algorithm = df_swc.algorithm
df_neuron_distance.loc[idx] = [df_swc.swc_file, df_gold_image.swc_file, algorithm, nd['ave']]
idx = idx + 1
df_neuron_distance.to_csv(data_DIR + "/neuron_distance.r.csv")
###### plot##################################################
df_nd = pd.read_csv(data_DIR + "/neuron_distance.r.csv")
outputDir = data_DIR + "/neuron_dist_plots_r"
if not path.exists(outputDir):
os.mkdir(outputDir)
CASE_BY_CASE_PLOT = 1
if CASE_BY_CASE_PLOT:
images = np.unique(df_nd.gold_swc_file)
