def main(args):
logfile = args['logfile']
printlog = getattr(flow.Printlog(logfile=logfile), 'print_to_log')
main_path = "/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20201221_neural_weighted_behavior/"
response_files = [
os.path.join(main_path, file) for file in os.listdir(main_path)
if 'responses' in file
]
bbb.sort_nicely(response_files)
responses = []
for file in response_files:
responses.append(np.load(file))
responses = np.asarray(responses)
responses_split = np.reshape(responses, (49, 2000, 3, 500))
responses_fft = fft(responses_split, axis=-1)
responses_fft[:, :, :, 15:23] = 0
responses_fft[:, :, :, 475:485] = 0
responses_filtered = ifft(responses_fft, axis=-1)
to_fit = np.reshape(responses_filtered[:, :, :, :].real,
(49 * 2000, 3 * 500))
printlog('to_fit shape: {}'.format(to_fit.shape))
printlog('clustering.........')
model = AgglomerativeClustering(distance_threshold=0,
n_clusters=None,
memory=main_path,
linkage='ward')
model = model.fit(to_fit)
printlog('complete!')
def main():
# Will look for and get folder that contains __done__
# If multiple done folders, will get the oldest one (based on alphanumeric sorting)
imports_path = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/imports/build_queue'
#done_flag = '__done__'
print('Checking build queue.')
print('Time: {}'.format(datetime.datetime.now()))
queued_folders = []
for item in os.listdir(imports_path):
print('Found queued folder {}'.format(item))
#item_path = os.path.join(imports_path, item)
queued_folders.append(item)
if len(queued_folders) == 0:
print('No queued folders found. Raising SystemExit.')
raise SystemExit
else:
bbb.sort_nicely(queued_folders)
# strip flag and pass directory name
#folder_stripped = queued_folders[0].strip(done_flag)
#os.rename(queued_folders[0], folder_stripped)
#sleep(20) # sleep to give system time to rename folder.
folder_to_build = os.path.join(
os.path.split(imports_path)[0], queued_folders[0])
print('Passing control to build_fly.sh for folder {}'.format(
folder_to_build))
os.system('sbatch build_fly.sh {}'.format(folder_to_build))
os.remove(os.path.join(imports_path, queued_folders[0]))
def main(args):
logfile = args['logfile']
imports_path = args['imports_path']
printlog = getattr(flow.Printlog(logfile=logfile), 'print_to_log')
width = 120
printlog(F"{'Checking build queue':.<{width}}")
#printlog('Time: {}'.format(datetime.datetime.now()))
queued_folders = []
for item in os.listdir(imports_path):
queued_folders.append(item)
printlog(F"Found queued folders{str(queued_folders):.>{width-20}}")
if len(queued_folders) == 0:
printlog('No queued folders found. Raising SystemExit.')
raise SystemExit
else:
bbb.sort_nicely(queued_folders)
folder_to_build = os.path.join(
os.path.split(imports_path)[0], queued_folders[0])
print(folder_to_build)
printlog(F"Commencing processing of{folder_to_build:.>{width-24}}")
printlog(f"{'> '+str(queued_folders[0])+' <':-^{width}}")
def main(args):
### Move folders from imports to fly dataset - need to restructure folders ###
logfile = args['logfile']
flagged_dir = args['flagged_dir']
target_path = args['dataset_path']
fly_dirs = args['fly_dirs']
printlog = getattr(flow.Printlog(logfile=logfile), 'print_to_log')
#printlog('\nBuilding flies from directory {}'.format(flagged_dir))
width = 120
#printlog(F"\n{'#' * width}\n"
# F"{' Building flies from directory ' + os.path.split(flagged_dir)[-1] + ' ':#^{width}}\n"
# F"{'#' * width}")
# Assume this folder contains fly1 etc
# This folder may (or may not) contain separate areas # False, now enforcing experiment subfolders
# Each area will have a T and a Z
# Avoid grabbing other weird xml files, reference folder etc.
# Need to move into fly_X folder that reflects it's date
# get fly folders in flagged directory and sort to ensure correct fly order
likely_fly_folders = os.listdir(flagged_dir)
bbb.sort_nicely(likely_fly_folders)
likely_fly_folders = [i for i in likely_fly_folders if 'fly' in i]
printlog(F"Found fly folders{str(likely_fly_folders):.>{width-17}}")
if fly_dirs is not None:
likely_fly_folders = fly_dirs
printlog(F"Continuing with only{str(likely_fly_folders):.>{width-20}}")
for likely_fly_folder in likely_fly_folders:
if 'fly' in likely_fly_folder:
new_fly_number = get_new_fly_number(target_path)
#printlog(f'\n*Building {likely_fly_folder} as fly number {new_fly_number}*')
printlog(
f"\n{' Building '+likely_fly_folder+' as fly_'+ str(new_fly_number) + ' ':-^{width}}"
)
# Define source fly directory
source_fly = os.path.join(flagged_dir, likely_fly_folder)
# Define destination fly directory
#fly_time = get_fly_time(source_fly)
new_fly_folder = 'fly_' + str(new_fly_number)
destination_fly = os.path.join(target_path, new_fly_folder)
os.mkdir(destination_fly)
printlog(F'Created fly directory:{destination_fly:.>{width-22}}')
# Copy fly data
copy_fly(source_fly, destination_fly, printlog)
# Add date to fly.json file
add_date_to_fly(destination_fly)
# Add json metadata to master dataset
add_fly_to_xlsx(destination_fly)
def main(args):
### Move folders from imports to fly dataset - need to restructure folders ###
flagged_directory = args[0]
print('Building fly from directory {}'.format(flagged_directory))
sys.stdout.flush()
imports_path = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/imports'
target_path = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20190101_walking_dataset/'
# Assume this folder contains fly_1 etc
# This folder may (or may not) contain separate areas # False, now enforcing experiment subfolders
# Each area will have a T and a Z
# Avoid grabbing other weird xml files, reference folder etc.
# Need to move into fly_X folder that reflects it's date
# Get new destination fly number by looking at last 2 char of current flies
current_fly_number = get_new_fly_number(target_path)
# get fly folders in flagged directory and sort to ensure correct fly order
likely_fly_folders = os.listdir(flagged_directory)
print('Found fly folders: {}'.format(likely_fly_folders))
sys.stdout.flush()
bbb.sort_nicely(likely_fly_folders)
for likely_fly_folder in likely_fly_folders:
if 'fly' in likely_fly_folder:
print('This fly will be number: {}'.format(current_fly_number))
print('Creating fly from directory: {}'.format(likely_fly_folder))
sys.stdout.flush()
# Define source fly directory
source_fly = os.path.join(flagged_directory, likely_fly_folder)
# Define destination fly directory
#fly_time = get_fly_time(source_fly)
new_fly_folder = 'fly_' + str(current_fly_number)
destination_fly = os.path.join(target_path, new_fly_folder)
os.mkdir(destination_fly)
print('Created fly directory: {}'.format(destination_fly))
sys.stdout.flush()
# Copy fly data
copy_fly(source_fly, destination_fly)
# Add date to fly.json file
add_date_to_fly(destination_fly)
# Add json metadata to master dataset
add_fly_to_xlsx(destination_fly)
# Get new fly number
current_fly_number += 1
def main():
root_directory = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20190101_walking_dataset/'
fly_folders = [os.path.join(root_directory,x) for x in os.listdir(root_directory) if 'fly' in x and int(x.split('_')[-1]) in [1,2,3,4,5,6,7,8,9]]
bbb.sort_nicely(fly_folders)
fly_folders = fly_folders[::-1]
#fly_folders = [os.path.join(root_directory, 'fly_1')]
for fly in fly_folders:
expt_folders = []
expt_folders = [os.path.join(fly,x) for x in os.listdir(fly) if 'func' in x]
if len(expt_folders) > 0:
for expt_folder in expt_folders:
xml_file = os.path.join(expt_folder, 'imaging', 'functional.xml')
create_imaging_json(xml_file)
def main():
print('Performing X on all flies')
root_directory = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20190101_walking_dataset/'
fly_folders = [
os.path.join(root_directory, x) for x in os.listdir(root_directory)
if 'fly' in x
]
bbb.sort_nicely(fly_folders)
fly_folders = fly_folders[::-1]
#fly_folders = [os.path.join(root_directory, 'fly_52')]
for fly in fly_folders:
expt_folders = []
expt_folders = [
os.path.join(fly, x) for x in os.listdir(fly) if 'func' in x
]
if len(expt_folders) > 0:
for expt_folder in expt_folders:
create_imaging_json(
os.path.join(expt_folder, 'imaging', 'functional.xml'))
def add_date_to_fly(destination_fly):
''' get date from xml file and add to fly.json'''
### Get date
try: # Check if there are func folders
# Get func folders
func_folders = [
os.path.join(destination_fly, x)
for x in os.listdir(destination_fly) if 'func' in x
]
bbb.sort_nicely(func_folders)
func_folder = func_folders[0]
# Get full xml file path
xml_file = os.path.join(func_folder, 'imaging', 'functional.xml')
except: # Use anatomy folder
# Get anat folders
anat_folders = [
os.path.join(destination_fly, x)
for x in os.listdir(destination_fly) if 'anat' in x
]
bbb.sort_nicely(anat_folders)
anat_folder = anat_folders[0]
# Get full xml file path
xml_file = os.path.join(anat_folder, 'imaging', 'anatomy.xml')
# Extract datetime
datetime_str, _, _ = get_datetime_from_xml(xml_file)
# Get just date
date = datetime_str.split('-')[0]
time = datetime_str.split('-')[1]
### Add to fly.json
json_file = os.path.join(destination_fly, 'fly.json')
with open(json_file, 'r+') as f:
metadata = json.load(f)
metadata['date'] = str(date)
metadata['time'] = str(time)
f.seek(0)
json.dump(metadata, f, indent=4)
f.truncate()
def main(args):
filename = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20190101_walking_dataset/master_2P.xlsx'
wb = load_workbook(filename=filename, read_only=False)
ws = wb.active
root_directory = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20190101_walking_dataset/'
if args[0] == 'all':
fly_folders = [
os.path.join(root_directory, x) for x in os.listdir(root_directory)
if 'fly' in x
]
bbb.sort_nicely(fly_folders)
else:
fly_folders = [os.path.join(root_directory, 'fly_' + str(args[0]))]
for fly_folder in fly_folders:
print(fly_folder)
# If no fly.json, just skip (atleast one fly in empty for some reason)
try:
fly_file = os.path.join(fly_folder, 'fly.json')
fly_data = load_json(fly_file)
except:
continue
expt_folders = []
expt_folders = [
os.path.join(fly_folder, x) for x in os.listdir(fly_folder)
if 'func' in x
]
bbb.sort_nicely(expt_folders)
for expt_folder in expt_folders:
expt_file = os.path.join(expt_folder, 'expt.json')
expt_data = load_json(expt_file)
# Occasionally a fly may not have an imaging folder (if only fictrac was recorded for example)
try:
scan_file = os.path.join(expt_folder, 'imaging', 'scan.json')
scan_data = load_json(scan_file)
scan_data['x_voxel_size'] = '{:.1f}'.format(
scan_data['x_voxel_size'])
scan_data['y_voxel_size'] = '{:.1f}'.format(
scan_data['y_voxel_size'])
scan_data['z_voxel_size'] = '{:.1f}'.format(
scan_data['z_voxel_size'])
except:
scan_data['laser_power'] = None
scan_data['PMT_green'] = None
scan_data['PMT_red'] = None
scan_data['x_dim'] = None
scan_data['y_dim'] = None
scan_data['z_dim'] = None
scan_data['x_voxel_size'] = None
scan_data['y_voxel_size'] = None
scan_data['z_voxel_size'] = None
visual_file = os.path.join(expt_folder, 'visual', 'visual.json')
try:
visual_data = load_json(visual_file)
visual_input = visual_data[0]['name'] + ' ({})'.format(
len(visual_data))
except:
visual_input = None
# Get fly_id
fly_folder = os.path.split(os.path.split(expt_folder)[0])[-1]
fly_id = fly_folder.split('_')[-1]
# Get expt_id
expt_id = expt_folder.split('_')[-1]
# Append the new row
new_row = []
new_row = [
int(fly_id),
int(expt_id), fly_data['date'], expt_data['brain_area'],
fly_data['genotype'], visual_input, None, fly_data['notes'],
expt_data['notes'], expt_data['time'],
fly_data['circadian_on'], fly_data['circadian_off'],
fly_data['gender'], fly_data['age'], fly_data['temp'],
scan_data['laser_power'], scan_data['PMT_green'],
scan_data['PMT_red'], scan_data['x_dim'], scan_data['y_dim'],
scan_data['z_dim'], scan_data['x_voxel_size'],
scan_data['y_voxel_size'], scan_data['z_voxel_size']
]
#if visual_data is not None:
# new_row['visual_stimuli'] = visual_data
#if fictrac_data is not None:
# new_row['fictrac'] = fictrac
ws.append(new_row)
# Save the file
filename = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20190101_walking_dataset/master_2P.xlsx'
wb.save(filename)
def main(args):
logfile = args['logfile']
printlog = getattr(flow.Printlog(logfile=logfile), 'print_to_log')
n_clusters = 2000
labels_file = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20201129_super_slices/cluster_labels.npy'
cluster_model_labels = np.load(labels_file)
################################
### Load and correct filters ###
################################
main_path = "/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20210316_neural_weighted_behavior"
response_files = [
os.path.join(main_path, file) for file in os.listdir(main_path)
if 'responses' in file
]
bbb.sort_nicely(response_files)
responses = []
for file in response_files:
responses.append(np.load(file))
responses = np.asarray(responses)
responses.shape
responses_split = np.reshape(responses, (49 - 18, 2000, 4, 500))
responses_fft = fft(responses_split, axis=-1)
responses_fft[:, :, :, 15:23] = 0
responses_fft[:, :, :, 475:485] = 0
responses_filtered = ifft(responses_fft, axis=-1)
responses_filtered = responses_filtered.real
#################
### NORMALIZE ###
#################
all_signals = np.reshape(responses_filtered[:, :, :, :],
(31 * 2000, 4, 500))
# (62000,4,500)
# want to exclude signals that have no response to any behavior
# each behavior type will have a different distribution of sum(abs(signal))
all_sums = np.sum(np.abs(all_signals), axis=-1)
# (62000,4)
# how to pick the threshold for each behavior?
# lets aim to use corr map sig threshold,
# but for now take top 25%
thresholds = np.percentile(all_sums, 75, axis=0)
all_signals[(all_sums < thresholds[np.newaxis, :])] = 0
all_maxs = np.max(np.abs(all_signals), axis=-1)
all_signals_normalized = all_signals / all_maxs[:, :, np.newaxis]
all_signals_normalized = np.nan_to_num(all_signals_normalized)
to_fit = np.reshape(all_signals_normalized, (62000, 2000))
###############
### CLUSTER ###
###############
t0 = time.time()
printlog('clustering.........')
model = AgglomerativeClustering(distance_threshold=0,
n_clusters=None,
memory=main_path,
linkage='ward')
model = model.fit(to_fit)
printlog('complete!')
printlog(str(time.time() - t0))
### Create linkage matrix for dendrogram
counts = np.zeros(model.children_.shape[0])
n_samples = len(model.labels_)
for i, merge in enumerate(model.children_):
current_count = 0
for child_idx in merge:
if child_idx < n_samples:
current_count += 1 # leaf node
else:
current_count += counts[child_idx - n_samples]
counts[i] = current_count
linkage_matrix = np.column_stack(
[model.children_, model.distances_, counts]).astype(float)
sys.setrecursionlimit(70000)
test = dendrogram(
linkage_matrix,
truncate_mode=None,
p=10,
color_threshold=None,
#link_color_func=lambda x: colors[x],
no_labels=True,
distance_sort=True,
no_plot=True)
printlog('did not fail!')
def main(args):
print('Stitcher started.')
directory = args.directory
print('directory: {}'.format(directory))
if args.datadir:
directory = datadir_appender.datadir_appender(directory)
# directory will contain motcorr_green_x.nii and motcorr_red_x.nii
# get list of reds and greens
reds = []
greens = []
for item in os.listdir(directory):
# sanity check that it is .nii
if '.nii' in item:
if 'red' in item:
reds.append(item)
elif 'green' in item:
greens.append(item)
# need to order correctly for correct stitching
bbb.sort_nicely(greens)
bbb.sort_nicely(reds)
# add directory path
reds = [os.path.join(directory, x) for x in reds]
greens = [os.path.join(directory, x) for x in greens]
if args.channels == 'rg':
colors = ['red', 'green']
channels = [reds, greens]
print('Using red and green channels.')
elif args.channels == 'r':
colors = ['red']
channels = [reds]
print('Using red channel.')
elif args.channels == 'g':
colors = ['green']
channels = [greens]
print('Using green channel.')
elif args.channels is None:
colors = ['red', 'green']
channels = [reds, greens]
print('Using red and green channels.')
### load brains ###
# This part in based on the input argparse
for i, channel in enumerate(channels):
brains = []
for brain_file in channel:
brain = bbb.load_numpy_brain(brain_file)
# Handle edgecase of single volume brain
if len(np.shape(brain)) == 3:
brain = brain[:, :, :, np.newaxis]
print('shape of partial brain: {}'.format(np.shape(brain)))
brains.append(brain)
print('brains len: {}'.format(len(brains)))
stitched_brain = np.concatenate(brains, axis=-1)
print('stitched_brain shape: {}'.format(np.shape(stitched_brain)))
save_file = os.path.join(directory,
'stitched_brain_{}.nii'.format(colors[i]))
bbb.save_brain(save_file, stitched_brain)
stitched_brain = None
# delete partial brains
[os.remove(file) for file in channel]
### Stitch motcorr params and create motcorr graph
# get motcorr param files
motcorr_param_files = []
for item in os.listdir(directory):
if '.npy' in item:
file = os.path.join(directory, item)
motcorr_param_files.append(file)
bbb.sort_nicely(motcorr_param_files)
# Load motcorr param files (needed to sort first)
motcorr_params = []
for file in motcorr_param_files:
motcorr_params.append(np.load(file))
if len(motcorr_params) > 0:
stitched_params = np.concatenate(motcorr_params, axis=0)
save_file = os.path.join(directory, 'motcorr_params_stitched')
np.save(save_file, stitched_params)
[os.remove(file) for file in motcorr_param_files]
xml_dir = os.path.join(os.path.split(directory)[0], 'imaging')
print('directory: {}'.format(directory))
print('xml_dir: {}'.format(xml_dir))
sys.stdout.flush()
bbb.save_motion_figure(stitched_params, xml_dir, directory)
else:
print('Empty motcorr params - skipping saving moco figure.')
### START Z-SCORING ###
os.system("sbatch zscore.sh {}".format(directory))
def add_fly_to_xlsx(fly_folder):
# Load xlsx
xlsx_path = '/oak/stanford/groups/trc/data/Brezovec/2P_Imaging/20190101_walking_dataset/master_2P.xlsx'
wb = load_workbook(filename=xlsx_path, read_only=False)
ws = wb.active
# If no fly.json, just skip (atleast one fly in empty for some reason)
try:
fly_file = os.path.join(fly_folder, 'fly.json')
fly_data = load_json(fly_file)
except:
return
expt_folders = []
expt_folders = [
os.path.join(fly_folder, x) for x in os.listdir(fly_folder)
if 'func' in x
]
bbb.sort_nicely(expt_folders)
for expt_folder in expt_folders:
expt_file = os.path.join(expt_folder, 'expt.json')
expt_data = load_json(expt_file)
# Occasionally a fly may not have an imaging folder (if only fictrac was recorded for example)
try:
scan_file = os.path.join(expt_folder, 'imaging', 'scan.json')
scan_data = load_json(scan_file)
scan_data['x_voxel_size'] = '{:.1f}'.format(
scan_data['x_voxel_size'])
scan_data['y_voxel_size'] = '{:.1f}'.format(
scan_data['y_voxel_size'])
scan_data['z_voxel_size'] = '{:.1f}'.format(
scan_data['z_voxel_size'])
except:
scan_data['laser_power'] = None
scan_data['PMT_green'] = None
scan_data['PMT_red'] = None
scan_data['x_dim'] = None
scan_data['y_dim'] = None
scan_data['z_dim'] = None
scan_data['x_voxel_size'] = None
scan_data['y_voxel_size'] = None
scan_data['z_voxel_size'] = None
visual_file = os.path.join(expt_folder, 'visual', 'visual.json')
try:
visual_data = load_json(visual_file)
visual_input = visual_data[0]['name'] + ' ({})'.format(
len(visual_data))
except:
visual_input = None
# Get fly_id
fly_folder = os.path.split(os.path.split(expt_folder)[0])[-1]
fly_id = fly_folder.split('_')[-1]
# Get expt_id
expt_id = expt_folder.split('_')[-1]
# Append the new row
new_row = []
new_row = [
int(fly_id),
int(expt_id), fly_data['date'], expt_data['brain_area'],
fly_data['genotype'], visual_input, None, fly_data['notes'],
expt_data['notes'], expt_data['time'], fly_data['circadian_on'],
fly_data['circadian_off'], fly_data['gender'], fly_data['age'],
fly_data['temp'], scan_data['laser_power_max'],
scan_data['PMT_green'], scan_data['PMT_red'], scan_data['x_dim'],
scan_data['y_dim'], scan_data['z_dim'], scan_data['x_voxel_size'],
scan_data['y_voxel_size'], scan_data['z_voxel_size']
]
ws.append(new_row)
# Save the file
wb.save(xlsx_path)