def debug(container_id, local=False, plots=None):
SCRIPT = "/data/informatics/CAM/analysis/allensdk/allensdk/internal/pipeline_modules/run_observatory_container_thumbnails.py"
SDK_PATH = "/data/informatics/CAM/analysis/allensdk/"
OUTPUT_DIR = "/data/informatics/CAM/analysis/containers"
container_dir = os.path.join(OUTPUT_DIR, str(container_id))
input_data = []
for exp in get_container_info(container_id):
exp_data = robsth.get_input_data(exp['id'])
exp_input_json = os.path.join(exp_data["output_directory"],
"input.json")
input_data.append(
dict(input_json=exp_input_json,
output_json=os.path.join(exp_data["output_directory"],
"output.json")))
Manifest.safe_make_parent_dirs(exp_input_json)
ju.write(exp_input_json, exp_data)
run_module(SCRIPT,
input_data,
container_dir,
sdk_path=SDK_PATH,
pbs=dict(vmem=32,
job_name="cthumbs_%d" % container_id,
walltime="10:00:00"),
local=local,
optional_args=['--types=' + ','.join(plots)] if plots else None)
def debug(experiment_id, local=False):
OUTPUT_DIRECTORY = "/data/informatics/CAM/neuropil"
SDK_PATH = "/data/informatics/CAM/neuropil/allensdk"
SCRIPT = "/data/informatics/CAM/neuropil/allensdk/allensdk/internal/pipeline_modules/run_neuropil_correction.py"
exp = lu.query("select * from ophys_experiments where id = %d" %
experiment_id)[0]
sd = exp["storage_directory"]
test_file = "/data/informatics/CAM/demix/%d/demixed_traces.h5" % experiment_id
if os.path.exists(test_file):
roi_trace_file = test_file
else:
roi_trace_file = os.path.join(sd, "demix",
"%d_demixed_traces.h5" % experiment_id)
exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
input_data = dict(roi_trace_file=roi_trace_file,
neuropil_trace_file=os.path.join(sd, "processed",
"neuropil_traces.h5"),
storage_directory=exp_dir)
run_module(SCRIPT,
input_data,
exp_dir,
sdk_path=SDK_PATH,
pbs=dict(vmem=160,
job_name="np_%d" % experiment_id,
walltime="36:00:00"),
local=local,
optional_args=['--log-level', 'DEBUG'])
def debug(experiment_id, local=False, raw_path=None):
OUTPUT_DIRECTORY = "/data/informatics/CAM/ophys_decomp"
SDK_PATH = "/data/informatics/CAM/ophys_decomp/allensdk"
SCRIPT = ("/data/informatics/CAM/ophys_decomp/allensdk/allensdk/"
"internal/pipeline_modules/run_ophys_session_decomposition.py")
exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
if raw_path is not None:
conversion_definitions = create_fake_metadata(exp_dir, raw_path)
input_data = {
"raw_filename": raw_path,
"frame_metadata": conversion_definitions
}
else:
raise NotImplementedError("No real examples exist yet")
run_module(SCRIPT,
input_data,
exp_dir,
sdk_path=SDK_PATH,
pbs=dict(vmem=160,
job_name="ophys_decomp_%d" % experiment_id,
walltime="36:00:00"),
local=local)
def debug(experiment_ids,
local=False,
OUTPUT_DIR="/data/informatics/CAM/analysis/",
SDK_PATH="/data/informatics/CAM/analysis/allensdk/",
walltime="10:00:00",
python=SHARED_PYTHON,
queue='braintv'):
input_data = {}
for eid in experiment_ids:
exp_dir = os.path.join(OUTPUT_DIR, str(eid))
input_data[eid] = dict(nwb_file=get_experiment_nwb_file(eid),
output_file=os.path.join(
exp_dir, "%d_analysis.h5" % eid),
session_name=get_experiment_session(eid))
run_module(os.path.abspath(__file__),
input_data,
exp_dir,
python=python,
sdk_path=SDK_PATH,
pbs=dict(vmem=32,
job_name="bobanalysis_%d" % eid,
walltime=walltime,
queue=queue),
local=local)
def debug(experiment_id,
local=False,
sdk_path=DEBUG_SDK_PATH,
script=DEBUG_SCRIPT,
output_directory=DEBUG_OUTPUT_DIRECTORY):
input_data = create_input_data(experiment_id)
exp_dir = os.path.join(output_directory, str(experiment_id))
run_module(script, input_data, exp_dir, sdk_path=sdk_path, local=local)
def debug(experiment_id, plots=None, local=False):
SDK_PATH = "/data/informatics/CAM/analysis/allensdk/"
input_data = get_input_data(experiment_id)
run_module(os.path.abspath(__file__),
input_data,
input_data["output_directory"],
sdk_path=SDK_PATH,
pbs=dict(vmem=32,
job_name="bobthumbs_%d"% experiment_id,
walltime="10:00:00"),
local=local,
optional_args=['--types='+','.join(plots)] if plots else None)
def debug(region_id, storage_directory="./", local=True,
sdk_path=SDK_PATH, script_path=SCRIPT_PATH, lims_host="lims2"):
strategy_class = "AnnotatedRegionMetricsStrategy"
object_class = "AnnotatedRegion"
input_json = get_input_json(region_id, object_class, strategy_class,
lims_host)
exp_dir = os.path.join(storage_directory, str(region_id))
run_module(script_path,
input_json,
exp_dir,
sdk_path=sdk_path,
pbs=dict(vmem=4,
job_name="isi_metrics_{}".format(region_id),
walltime="1:00:00"),
local=local)
def debug(experiment_id, num_frames=None, threshold_factor=None, local=False):
OUTPUT_DIR = "/data/informatics/CAM/eye_tracking/"
SDK_PATH = "/data/informatics/CAM/eye_tracking/allensdk"
SCRIPT_PATH = "/data/informatics/CAM/eye_tracking/allensdk/allensdk/internal/pipeline_modules/run_eye_tracking.py"
experiment_dir = os.path.join(OUTPUT_DIR, str(experiment_id))
info = get_experiment_info(experiment_id)
info['output_directory'] = experiment_dir
optional_args = []
if num_frames is not None:
optional_args += ['--num_frames', str(num_frames)]
run_module(SCRIPT_PATH,
info,
experiment_dir,
sdk_path=SDK_PATH,
pbs=dict(vmem=160,
job_name="itrack_%d" % experiment_id,
walltime="10:00:00"),
local=local,
optional_args=optional_args)
def debug(experiment_id, local=False):
OUTPUT_DIRECTORY = "/data/informatics/CAM/demix"
SDK_PATH = "/data/informatics/CAM/analysis/allensdk"
SCRIPT = "/data/informatics/CAM/analysis/allensdk/allensdk/internal/pipeline_modules/run_demixing.py"
sd = lu.query(
"select storage_directory from ophys_experiments where id = %d" %
experiment_id)[0]['storage_directory']
rois = lu.query("select * from cell_rois where ophys_experiment_id = %d" %
experiment_id)
exc_labels = lu.query("""
select cr.id, rel.name as exclusion_label from cell_rois cr
join cell_rois_roi_exclusion_labels crrel on crrel.cell_roi_id = cr.id
join roi_exclusion_labels rel on crrel.roi_exclusion_label_id = rel.id
where cr.ophys_experiment_id = %d
""" % experiment_id)
nrois = {
roi['id']: dict(width=roi['width'],
height=roi['height'],
x=roi['x'],
y=roi['y'],
id=roi['id'],
valid=roi['valid_roi'],
mask=roi['mask_matrix'],
exclusion_labels=[])
for roi in rois
}
for exc_label in exc_labels:
nrois[exc_label['id']]['exclusion_labels'].append(
exc_label['exclusion_label'])
movie_path_response = lu.query('''
select wkf.filename, wkf.storage_directory from well_known_files wkf
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where wkf.attachable_id = {} and wkf.attachable_type = 'OphysExperiment'
and wkft.name = 'MotionCorrectedImageStack'
'''.format(experiment_id))
movie_h5_path = os.path.join(movie_path_response[0]['storage_directory'],
movie_path_response[0]['filename'])
exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
input_data = {
"movie_h5": movie_h5_path,
"traces_h5": os.path.join(sd, "processed", "roi_traces.h5"),
"roi_masks": nrois.values(),
"output_file": os.path.join(exp_dir, "demixed_traces.h5")
}
run_module(SCRIPT,
input_data,
exp_dir,
sdk_path=SDK_PATH,
pbs=dict(vmem=160,
job_name="demix_%d" % experiment_id,
walltime="36:00:00"),
local=local,
optional_args=['--log-level', 'DEBUG'])
def debug(experiment_id, local=False):
OUTPUT_DIRECTORY = "/data/informatics/CAM/eye_calibration"
SDK_PATH = "/data/informatics/CAM/eye_calibration/allensdk"
SCRIPT = ("/data/informatics/CAM/eye_calibration/allensdk/allensdk"
"/internal/pipeline_modules/run_ophys_eye_calibration.py")
frame_width = 640
frame_height = 480
cr_file = get_wkf("EyeTracking Corneal Reflection", experiment_id)
pupil_file = get_wkf("EyeTracking Pupil", experiment_id)
exp_info = lu.query("""
select *
from ophys_sessions os
where os.id = {}
""".format(experiment_id))[0]
exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
# clear out missing values to let us get defaults
for key, value in list(exp_info.items()):
if value is None:
del exp_info[key]
input_data = {
"cr_params_file": cr_file,
"pupil_params_file": pupil_file,
"frame_width": frame_width,
"frame_height": frame_height,
"output_file": os.path.join(exp_dir,
"eye_tracking_to_screen_mapping.h5"),
"monitor_position_x_mm": exp_info.get(
"screen_center_x_mm", eye_calibration.MONITOR_POSITION_OLD[0]*10),
"monitor_position_y_mm": exp_info.get(
"screen_center_y_mm", eye_calibration.MONITOR_POSITION_OLD[1]*10),
"monitor_position_z_mm": exp_info.get(
"screen_center_z_mm", eye_calibration.MONITOR_POSITION_OLD[2]*10),
"monitor_rotation_x_deg": exp_info.get("screen_rotation_x_deg", 0),
"monitor_rotation_y_deg": exp_info.get("screen_rotation_y_deg", 0),
"monitor_rotation_z_deg": exp_info.get("screen_rotation_z_deg", 0),
"camera_position_x_mm": exp_info.get(
"camera_center_x_mm", eye_calibration.CAMERA_POSITION_OLD[0]*10),
"camera_position_y_mm": exp_info.get(
"camera_center_y_mm", eye_calibration.CAMERA_POSITION_OLD[1]*10),
"camera_position_z_mm": exp_info.get(
"camera_center_z_mm", eye_calibration.CAMERA_POSITION_OLD[2]*10),
"camera_rotation_x_deg": exp_info.get(
"camera_rotation_x_deg",
eye_calibration.CAMERA_ROTATIONS_OLD[0]*180/np.pi),
"camera_rotation_y_deg": exp_info.get(
"camera_rotation_y_deg",
eye_calibration.CAMERA_ROTATIONS_OLD[1]*180/np.pi),
"camera_rotation_z_deg": exp_info.get(
"camera_rotation_z_deg",
eye_calibration.CAMERA_ROTATIONS_OLD[2]*180/np.pi),
"led_position_x_mm": exp_info.get(
"led_center_x_mm", eye_calibration.LED_POSITION_ORIGINAL[0]*10),
"led_position_y_mm": exp_info.get(
"led_center_y_mm", eye_calibration.LED_POSITION_ORIGINAL[1]*10),
"led_position_z_mm": exp_info.get(
"led_center_z_mm", eye_calibration.LED_POSITION_ORIGINAL[2]*10)
}
# TEMPORARY HACKS TO DEAL WITH BAD DATA IN LIMS
# TODO: REMOVE WHEN DATAFIXES DONE
if input_data["monitor_position_x_mm"] == -86.2:
input_data["monitor_position_x_mm"] = \
eye_calibration.MONITOR_POSITION_NEW[0]*10
input_data["monitor_position_y_mm"] = \
eye_calibration.MONITOR_POSITION_NEW[1]*10
input_data["monitor_position_z_mm"] = \
eye_calibration.MONITOR_POSITION_NEW[2]*10
run_module(SCRIPT,
input_data,
exp_dir,
sdk_path=SDK_PATH,
local=local)