def stats_calculator(ippath, oppath):
imglist = [x for x in os.listdir(ippath) if not x.startswith('.')]
var = ['df_nodes', 'df_points', 'df_segments']
counts_combined = []
names = []
for img in imglist:
filelist, fileabslist = ListFiles(os.path.join(ippath, img),
extension='.csv')
df_points = pd.read_csv(os.path.join(ippath, img, 'points.csv'))
df_segments = pd.read_csv(os.path.join(ippath, img, 'segments.csv'))
df_nodes = pd.read_csv(os.path.join(ippath, img, 'nodes.csv'))
opfilename = 'segments_s.csv'
countfilename = 'count.csv'
countfilename_combined = 'counts_combined.csv'
if opfilename not in filelist:
df_segments_s = SegStats(df_points, df_segments)
df_segments_s.to_csv(os.path.join(oppath, img, opfilename),
index=False)
counts = (PNSCount(df_points, df_nodes, df_segments))
counts_combined.append(counts)
names.append(img)
fileinfo = pd.read_csv(os.path.join('./par', 'lung_file_idx.csv'))
print(names)
img_group = []
img_treatment = []
for i in names:
img_group.append(
fileinfo[fileinfo['data_filename'] == i]['genotype'].item())
img_treatment.append(
fileinfo[fileinfo['data_filename'] == i]['treatment'].item())
if countfilename_combined not in imglist:
df_counts_combined = pd.DataFrame(
counts_combined, columns=['Points', 'Nodes', 'Segments'])
df_counts_combined['Names'] = names
df_counts_combined['Genotype'] = img_group
df_counts_combined['Treatment'] = img_treatment
df_counts_combined.to_csv(os.path.join(path, countfilename_combined),
index=False)
def convert_xml_csv(ippath, oppath):
filelist, fileabslist = ListFiles(ippath, extension='.xml')
for idx, f in enumerate(filelist):
filename = f.replace('.xml', '')
ip = os.path.join(ippath, f)
op = os.path.join(oppath, filename)
print(ip)
print(op)
# create path
if filename not in os.listdir(oppath):
DirCheck(op)
# convert *.xml to *.csv
csv_all = MSXmlReader(ip)
# save each spreadsheet into individual *.csv file
for key, value in csv_all.items():
oppath_tmp = os.path.join(op, key + '.csv')
value.to_csv(oppath_tmp, index=False)
# create grouped filename and file path
colors = {'wildtype': 'red', 'knockout': 'blue'}
group = {'wildtype': 'w', 'knockout': 'k'}
ippath = {
'stormdata': {
'dir': stormcsv_path,
'ext': '.csv',
}
}
oppath = {'histogram': {'dir': photon_histo_path, 'ext': '.png'}}
# %%
# create filenamelist
filenamelist = ListFiles(os.path.join(stormcsv_path, str(1)),
'.csv')['filelist']
mainfilelist = GetGrpFLs(filenamelist, nchannel, group, ippath, oppath)
pprint.pprint(mainfilelist)
# ----------------------------------------------------- #
# %%
data_list = []
for c in range(nchannel):
for g in group.keys():
for i in range(len(mainfilelist[str(c + 1)][g]['filename_ip'])):
# for i in range(10):
filepath = mainfilelist[str(c + 1)][g]['stormdata'][i]
# print(filepath)
data = pd.read_csv(filepath, header=0, index_col=0)
data['filename'] = mainfilelist[str(c + 1)][g]['filename_ip'][i]
data['group'] = g
'nnddata':{
'dir': nnd_data_path,
'ext': '.csv',
}
}
oppath = {
'summary': {
'dir': nnd_plot_path,
'ext': '.png'
}
}
# %%
# create filenamelist
filenamelist = ListFiles(os.path.join(nnd_data_path, str(1)), '.csv')['filelist']
mainfilelist = GetGrpFLs(filenamelist, nchannel, group, ippath, oppath)
pprint.pprint(mainfilelist)
# concate the data into one dataframe
data_list = []
for c in range(nchannel):
for g in group.keys():
for i in range(len(mainfilelist[str(c+1)][g]['filename_ip'])):
filepath = mainfilelist[str(c+1)][g]['nnddata'][i]
data = pd.read_csv(filepath, header=0, index_col = 0)
data['filename'] = mainfilelist[str(c+1)][g]['filename_ip'][i]
data['group'] = g
data['channel'] = str(c+1)
data_list.append(data)
# ========================================================
#%%
path = '/Volumes/LaCie_DataStorage/Asensio_Lab'
raw_dir = 'alignment_opencv_01'
sub_dir = 'KO_2_reordered_rename'
ippath = os.path.join(path, raw_dir, sub_dir)
print(ippath)
op_dir = 'alignment_opencv_01_std'
oppath = os.path.join(path, op_dir, sub_dir)
DirCheck(oppath)
#%%
filename, filepath = ListFiles(ippath, '.tif')
print(filepath)
#%%
inten_min = 0.0
inten_max = 0.0
for i in range(len(filename)):
ipfilepath = os.path.join(ippath, filename[i] + '.tif')
opfilepath = os.path.join(oppath, filename[i] + '.tif')
ROI = [4352, 3088, 4096, 4096]
im = Image.open(ipfilepath)
im_std = Standardize(im, ROI)
if inten_min > np.amin(im_std):
inten_min = np.amin(im_std)
'stormdata':{
'dir': stormcsv_path,
'ext': '.csv',
}
}
oppath = {
'stormdata_th': {
'dir': csv_th_path,
'ext': '.csv'
}
}
# %%
# create filenamelist
filenamelist = ListFiles(os.path.join(stormcsv_path, str(1)), '.csv')['filelist']
mainfilelist = GetGrpFLs(filenamelist, nchannel, group, ippath, oppath)
pprint.pprint(mainfilelist)
#%%
# export sliced csv
threshold = {
'1': threshold_photon_c1,
'2': threshold_photon_c2,
}
data_list = []
for c in range(nchannel):
for g in group.keys():
for i in range(len(mainfilelist[str(c+1)][g]['filename_ip'])):
# for i in range(10):
# ========================================================
#%%
path = '/Volumes/LaCie_DataStorage/Asensio_Lab'
raw_dir = 'raw'
sub_dir = 'KO_2'
raw_path = os.path.join(path, raw_dir, sub_dir)
print(raw_path)
op_dir = 'output'
oppath = os.path.join(path, op_dir, sub_dir)
DirCheck(oppath)
#%%
filename, filepath = ListFiles(raw_path, '.tif')
print(filepath)
#%%
cleaned_filelist = SortByFolder(filepath, 'Tile_')
print([item.filename for item in cleaned_filelist])
#%%
# copy files to the new output folder
for item in cleaned_filelist:
print(item.filename)
copyfile(item.filepath, os.path.join(oppath, 'image_' + str(item.index).zfill(4) + '.tif'))
#%%
foldernames = ListFolders(ippath)
dirchecklist = []
for i in foldernames:
dirchecklist.append(os.path.join(oppath, i))
DirCheck(dirchecklist)
#%%
par_path = os.path.join(path, project_dir, 'par', 'tile_dimension.csv')
par_data= pd.read_csv(par_path)
display(par_data)
#%%
filelist, fileabslist = ListFiles(ippath, extension = '.mrc')
# print(filelist)
org_filename = par_data['cellname']
class imgfile:
def __init__(self, filename, filepath, foldername, folderpath):
self.filename = filename
self.filepath = filepath
self.foldername = foldername
self.folderpath = folderpath
filelistprofile = []
for idx in range(len(fileabslist)):
path_tmp = fileabslist[idx]
filename_tmp = filelist[idx]
folderpath_tmp = os.path.split(path_tmp)[0]
# style.use('fivethirtyeight')
from core.fileop import DirCheck, ListFiles
from core.filamentanalysis import SegStats, BranchLabel
#%%
ippath = '/Volumes/LaCie_DataStorage/Woo-lungs/2019/data/normoxia/'
var = ['df_nodes', 'df_points', 'df_segments']
fl_names = ['lung_vs_002_nodes.csv', 'lung_vs_002_points.csv', 'lung_vs_002_segments.csv']
for idx in range(len(fl_names)):
exec("%s = pd.read_csv(os.path.join(ippath, '%s'))"%(var[idx], fl_names[idx]))
#%%
# calculate length for each segments
filelist = ListFiles(ippath, extension='.csv')[0]
filelist = [filename + '.csv' for filename in filelist]
print(filelist)
#%%
opfilename = 'lung_vs_002_segments_s.csv'
if opfilename in filelist:
df_segments_s = pd.read_csv(os.path.join(ippath, opfilename), header = 0)
else:
df_segments_s = SegStats(df_points, df_segments)
df_segments_s.to_csv(os.path.join(ippath, opfilename), index = False)
#%%
display(df_segments_s)
#%%
def AlignImagesStack_Affine(ippath, oppath, oppath_match = None, factor = 1, warp_mode = cv2.MOTION_HOMOGRAPHY,
centerimg = None, startimg = None, endimg = None, ext = '.tif', GOOD_MATCH_PERCENT = 0.1):
# configure parameters
filename, filepath = ListFiles(ippath, ext)
# print(filepath)
if startimg is None:
startimg_idx = 0
else:
startimg_idx = startimg - 1
print("The First Image of the Stack: Slice No.{}(idx {})".format(startimg, startimg_idx))
if endimg is None:
endimg_idx = len(filename)
else:
endimg_idx = endimg - 1
print("The Last Image of the Stack: Slice No.{}(idx {})".format(endimg, endimg_idx))
if centerimg is None:
center_idx = startimg_idx
else:
center_idx = centerimg - 1
print("Image registration starting from: Slice No.{}(idx {})".format(centerimg, center_idx))
copyfile(filepath[center_idx], os.path.join(oppath, filename[center_idx] + ext))
# center to the end
for i in range(center_idx, endimg_idx):
print(i)
if filename[i] == filename[-1]:
break
im1_path = os.path.join(oppath, filename[i] + ext)
im2_path = filepath[i+1]
opfilename = filename[i+1]
imip_1, imip_2, im2Reg, h, imMatches = AlignImages_Affine(im1_path, im2_path, factor = factor, GOOD_MATCH_PERCENT = GOOD_MATCH_PERCENT)
im2Reg_pil = Image.fromarray(im2Reg)
im2Reg_pil.save(os.path.join(oppath, opfilename + ext))
if oppath_match is not None:
cv2.imwrite(os.path.join(oppath_match, opfilename + '.png'), imMatches)
# ceter to the start
for i in reversed(range(startimg_idx + 1, center_idx+1)):
print(i)
im1_path = os.path.join(oppath, filename[i] + ext)
im2_path = filepath[i-1]
opfilename = filename[i-1]
imip_1, imip_2, im2Reg, h, imMatches = AlignImages_Affine(im1_path, im2_path, factor = factor, GOOD_MATCH_PERCENT = GOOD_MATCH_PERCENT)
im2Reg_pil = Image.fromarray(im2Reg)
im2Reg_pil.save(os.path.join(oppath, opfilename + ext))
if oppath_match is not None:
cv2.imwrite(os.path.join(oppath_match, opfilename + '.png'), imMatches)
print(ippath_img_tmp)
DirCheck(oppath_img_tmp)
print(j)
m = re.search('^img_(0*[1-9][0-9]*)$', j)
if m:
found = m.group(1)
found = found.strip('0')
print(found)
filename_list = list(dimdata['filename'])
idx = filename_list.index(found)
print(idx)
tile_dim = (dimdata['x'][idx], dimdata['y'][idx])
box_dim = (3, 3)
filelist, fileabslist = ListFiles(ippath_img_tmp, '.tif')
print(filelist)
print(ippath_img_tmp)
# tile_dim = (10, 8)
print(tile_dim)
for idx in range(len(filelist)):
print(idx)
IdxConverter(idx, tile_dim, box_dim)
ip_filename = j + '_' + str(idx).zfill(4) + '.tif'
op_filename = j + '_' + str(IdxConverter(idx, tile_dim, box_dim)).zfill(4) + '.tif'
print(ip_filename)
ip_path = os.path.join(ippath, i, j, ip_filename)
print(op_filename)
op_path = os.path.join(oppath, i, j, op_filename)
# copyfile(ip_path, op_path)