def compute_gold_standard_distances(input_dir, gold_standard_swc):
subdirs = [x[0] for x in os.walk(input_dir)]
count = 0
for recon_dir in subdirs[1:]:
print recon_dir
folder_name = recon_dir.split('/')[-1]
cout = 0
cout_Cal = 0
for dirpath, dnames, fnames in os.walk(input_dir):
for f in fnames:
if f.endswith(".swc"):
swc_f = (os.path.join(dirpath, f))
siz = os.path.getsize(swc_f)
if (siz < 1024 * 1024):
log_file = swc_f + ".r.log"
if not os.path.exists(log_file):
bn.run_neuron_dist(swc_f, gold_standard_swc, log_file,
0, output_dir + "/nd")
cout_Cal = cout_Cal + 1
else:
print "skip big file:", swc_f
cout = cout + 1
print cout, " big files skipped"
print cout_Cal, " finished"
def run_neuron_distance():
os.system('rm ' +data_DIR+"/qsub_xx/*")
for recon_dir in subdirs[1:]:
folder_name = recon_dir.split('/')[-1]
if 'processed' in folder_name:
print recon_dir
files = glob.glob(recon_dir+'/../00_*strict.swc')
if len(files)>0:
gs_swc_file =files[0]
#auto recons
swc_files = []# glob.glob(recon_dir+'/*.strict.swc')
for i in range(len(swc_files)) :
input = swc_files[i]
out_fn=recon_dir+"/../processed/"+input.split('/')[-1]+'.dist.r.log'
print out_fn
if not os.path.exists(out_fn):
bn.run_neuron_dist(inputswc_path1=input, inputswc_path2=gs_swc_file, logfile= out_fn,
GEN_QSUB = 1, qsub_script_dir= data_DIR+"/qsub_xx", id=None)
#consensus
consensus_swc = recon_dir+'/../consensus3.strict.swc'
if path.exists(consensus_swc):
out_fn=consensus_swc+'.dist.r.log'
bn.run_neuron_dist(inputswc_path1=consensus_swc, inputswc_path2=gs_swc_file, logfile= out_fn,
GEN_QSUB = 1, qsub_script_dir= data_DIR+"/qsub_xx", id=None)
def cal_neuron_dist(input_csv_file,
output_csv,
overwrite_existing=1,
GEN_QSUB=0):
df_input = pd.read_csv(input_csv_file)
df_already_have = pd.DataFrame(columns=df_input.columns)
# if (not overwrite_existing):
# if os.path.isfile(old_output_csv):
# # only run new data
# df_old = pd.read_csv(old_output_csv)
# df_already_have = pd.merge(df_input, df_old, on='swc_file')
# print "there are already "+ str(df_already_have['swc_file'].size) +" swcs calculated"
output_dir = os.path.dirname(output_csv)
print " Calculate neuron distances:"
for i in range(df_input.image_file_name.size):
print "swc file :" + str(i)
swc_f = df_input.iloc[i].swc_file
log_file = df_input.iloc[i].swc_file + ".r.log"
#if not swc_f in list(df_already_have['swc_file'])
if overwrite_existing or not os.path.isfile(log_file):
bn.run_neuron_dist(swc_f, df_input.iloc[i].gold_swc_file, log_file,
GEN_QSUB, output_dir + "/nd")
#collect results from log files
df_neuron_distance = pd.DataFrame(
columns=('image_file_name', 'swc_file', 'gold_swc_file', 'algorithm',
'neuron_distance_12', 'neuron_distance_21',
'neuron_distance_ave', 'neuron_distance_diff',
'neuron_distance_perc'))
for i in range(df_input.image_file_name.size):
tmp = df_input.iloc[i].swc_file
logfile = tmp + ".r.log"
if path.isfile(logfile):
nd = bn.read_neuron_dist_log(logfile)
df_neuron_distance.loc[i] = [
df_input.iloc[i].image_file_name, df_input.iloc[i].swc_file,
df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm,
nd['dist_12'], nd['dist_21'], nd['ave'], nd['diff'], nd['perc']
]
else:
print "Warning: no neuron distance log output for " + tmp + " :output NAs."
df_neuron_distance.loc[i] = [
df_input.iloc[i].image_file_name, df_input.iloc[i].swc_file,
df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm,
np.nan, np.nan, np.nan, np.nan, np.nan
]
df_neuron_distance['neuron_difference'] = df_neuron_distance[
'neuron_distance_diff'] * df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_csv, index=False)
print "output:" + output_csv
return
def cal_neuron_dist(input_csv_file,output_csv,overwrite_existing = 1,GEN_QSUB = 0 ):
df_input = pd.read_csv(input_csv_file)
df_already_have = pd.DataFrame(columns = df_input.columns)
# if (not overwrite_existing):
# if os.path.isfile(old_output_csv):
# # only run new data
# df_old = pd.read_csv(old_output_csv)
# df_already_have = pd.merge(df_input, df_old, on='swc_file')
# print "there are already "+ str(df_already_have['swc_file'].size) +" swcs calculated"
output_dir = os.path.dirname(output_csv)
print " Calculate neuron distances:"
for i in range(df_input.image_file_name.size):
print "swc file :" + str(i)
swc_f = df_input.iloc[i].swc_file
log_file = df_input.iloc[i].swc_file + ".r.log"
#if not swc_f in list(df_already_have['swc_file'])
if overwrite_existing or not os.path.isfile(log_file) :
bn.run_neuron_dist(swc_f, df_input.iloc[i].gold_swc_file,log_file, GEN_QSUB, output_dir+"/nd")
#collect results from log files
df_neuron_distance = pd.DataFrame(columns=('image_file_name','swc_file', 'gold_swc_file',
'algorithm',
'neuron_distance_12','neuron_distance_21',
'neuron_distance_ave','neuron_distance_diff',
'neuron_distance_perc'))
for i in range(df_input.image_file_name.size):
tmp = df_input.iloc[i].swc_file
logfile = tmp + ".r.log"
if path.isfile(logfile):
nd = bn.read_neuron_dist_log(logfile)
df_neuron_distance.loc[i] = [df_input.iloc[i].image_file_name,df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, nd['dist_12'],nd['dist_21'],nd['ave'],nd['diff'],
nd['perc']]
else:
print "Warning: no neuron distance log output for "+tmp +" :output NAs."
df_neuron_distance.loc[i]= [df_input.iloc[i].image_file_name,df_input.iloc[i].swc_file, df_input.iloc[i].gold_swc_file, df_input.iloc[i].algorithm, np.nan, np.nan, np.nan, np.nan, np.nan]
df_neuron_distance['neuron_difference'] = df_neuron_distance['neuron_distance_diff'] *df_neuron_distance['neuron_distance_perc']
df_neuron_distance.to_csv(output_csv,index=False)
print "output:"+output_csv
return
def run_neuron_distance():
os.system('rm ' + data_DIR + "/qsub_xx/*")
for recon_dir in subdirs[1:]:
folder_name = recon_dir.split('/')[-1]
if 'processed' in folder_name:
print recon_dir
files = glob.glob(recon_dir + '/../00_*strict.swc')
if len(files) > 0:
gs_swc_file = files[0]
#auto recons
swc_files = [] # glob.glob(recon_dir+'/*.strict.swc')
for i in range(len(swc_files)):
input = swc_files[i]
out_fn = recon_dir + "/../processed/" + input.split(
'/')[-1] + '.dist.r.log'
print out_fn
if not os.path.exists(out_fn):
bn.run_neuron_dist(inputswc_path1=input,
inputswc_path2=gs_swc_file,
logfile=out_fn,
GEN_QSUB=1,
qsub_script_dir=data_DIR + "/qsub_xx",
id=None)
#consensus
consensus_swc = recon_dir + '/../consensus3.strict.swc'
if path.exists(consensus_swc):
out_fn = consensus_swc + '.dist.r.log'
bn.run_neuron_dist(inputswc_path1=consensus_swc,
inputswc_path2=gs_swc_file,
logfile=out_fn,
GEN_QSUB=1,
qsub_script_dir=data_DIR + "/qsub_xx",
id=None)
def compute_gold_standard_distances(input_dir,gold_standard_swc):
subdirs = [x[0] for x in os.walk(input_dir)]
count = 0
for recon_dir in subdirs[1:]:
print recon_dir
folder_name = recon_dir.split('/')[-1]
cout = 0
cout_Cal = 0
for dirpath, dnames, fnames in os.walk(input_dir):
for f in fnames:
if f.endswith(".swc"):
swc_f= (os.path.join(dirpath, f))
siz=os.path.getsize(swc_f)
if (siz<1024*1024):
log_file = swc_f + ".r.log"
if not os.path.exists(log_file):
bn.run_neuron_dist(swc_f, gold_standard_swc,log_file, 0, output_dir+"/nd")
cout_Cal=cout_Cal+1
else:
print "skip big file:",swc_f
cout= cout+1
print cout, " big files skipped"
print cout_Cal, " finished"
final_images = np.intersect1d(images_gold, images_results)
print "There are ", final_images.size, " images reconstructed in total"
#################################################
RUN_DISTANCE_CALC = 0
if RUN_DISTANCE_CALC:
for i in range(final_images.size):
imageName = final_images[i]
df_image = df_results[df_results.image == imageName]
df_gold_image = df_gold[df_gold.image == imageName].iloc[0]
for j in range(df_image.shape[0]):
df_swc = df_image.iloc[j]
# bn.run_neuron_dist(df_swc.swc_file, df_gold_image.swc_file, df_swc.swc_file+".log", 1, "nd")
bn.run_neuron_dist(df_gold_image.swc_file, df_swc.swc_file, df_swc.swc_file + ".r.log", 1, "nd")
###############################################
COLLECT_NEURON_DISTANCE = 1
if COLLECT_NEURON_DISTANCE:
df_neuron_distance = pd.DataFrame(columns=('swc_file', 'gold_swc_file', 'algorithm', 'neuron_distance'))
idx = 0
for i in range(final_images.size):
imageName = final_images[i]
df_image = df_results[df_results.image == imageName]
df_gold_image = df_gold[df_gold.image == imageName].iloc[0]
for j in range(df_image.shape[0]):
df_swc = df_image.iloc[j]