def analysis(self, string, dict, project_dir): # string = analysis if debug: util.print_dict_keys(dict) analysis_datasets = ["description", "good_trials", "trial_start_times",\ "trial_type_mat", "trial_type_string"] h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) ag = h5_object.make_group("analysis", abort=False) path = "/" + ag.name for k in dict.keys(): if not k.split(".")[-1] == "attrs": name = k.split(".")[-1] attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] try: ag.set_dataset(name, dict[k], attrs=attrs) except: ag.set_custom_dataset(name, dict[k], attrs=attrs) if not name in analysis_datasets: print " Warning: creating custom dataset analysis." + name if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def processing_extracellular_units_top_datasets(self, string, dict, project_dir): print "dict.keys()=", dict.keys() if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) module_name = "extracellular_units" mod = h5_object.make_group("<Module>", module_name, attrs={"ancestry": np.array(["Module>"])}) for k in dict.keys(): data_name = k.split(".")[-1] if not data_name == "attrs": attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if len(k.split(".")) == 3: try: mod.set_dataset(data_name, dict[k], attrsc=cattrs) except: mod.set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def general_subject(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) subject_datasets = ["age", "description", "genotype", "sex", "species", \ "subject_id", "weight"] h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) gg = h5_object.make_group("general", abort=False) sg = gg.make_group("subject", abort=False) for k in dict.keys(): data_name = k.split(".")[-1] if not data_name == "attrs": attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not data_name in subject_datasets: print " Warning: creating custom dataset general.subject." + data_name try: sg.set_dataset(data_name, dict[k], attrs=attrs) except: sg.set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def stimulus_presentation(self, string, dict, project_dir): h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) if debug: util.print_dict_keys(dict) series_path = "/stimulus/presentation" for k in dict.keys(): if len(k.split(".")) == 4: data_name = k.split(".")[-1] if not data_name == "attrs": group_name = k.split(".")[-2] # We assume that all groups have attributes group_attrs = dict[".".join([ k.split(".")[0], k.split(".")[1], group_name, "attrs" ])] series_type = group_attrs["series_type"] tsg = h5_object.make_group(series_type, group_name, path = series_path, \ attrs = group_attrs, abort=False) data_attrs = {} if k + ".attrs" in dict.keys(): data_attrs = dict[k + ".attrs"] try: tsg.set_dataset(data_name, dict[k], attrs=data_attrs) except: tsg.set_custom_dataset(data_name, dict[k], attrs=data_attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def general_top_datasets(self, string, dict, project_dir): # string = general.top_datasets if debug: util.print_dict_keys(dict) top_datasets = ["data_collection", "experiment_description", "experimenter", \ "institution", "lab", "notes", "pharmacology", "protocol", \ "reference_atlas", "related_publications", "session_id", \ "slices", "source_script", "stimulus", "surgery", \ "task_keywords", "whisker_configuration", "virus"] h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) gg = h5_object.make_group("general", abort=False) for k in dict.keys(): name = k.split(".")[len(k.split(".")) - 1] if not name in top_datasets: print " Warning: creating custom dataset general." + name attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] try: gg.set_dataset(name, dict[k], attrs=attrs) except: gg.set_custom_dataset(name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def acquisition_images(self, string, dict, project_dir): # string = acquisition.images if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) ag = h5_object.make_group("acquisition", abort=False) aig = ag.make_group("images", abort=False) for k in dict.keys(): if re.search("description", k): try: aig.set_dataset("description", dict[k]) except: aig.set_custom_dataset("description", dict[k]) elif not k.split(".")[-1] == "attrs": name = k.split(".")[-1] attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if re.search("fov_", k): h5_object.set_dataset("<image_X>", dict[k], name=name,\ dtype='uint8', attrs=attrs) else: try: aig.set_dataset(name, dict[k], attrs=attrs) except: aig.set_custom_dataset(name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def acquisition_timeseries_extracellular_traces(self, string, dict, project_dir): # string = acquisition.timeseries.extracellular_traces if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) group_attrs = {} if "acquisition.timeseries.extracellular_traces.attrs" in dict.keys(): group_attrs = dict[ "acquisition.timeseries.extracellular_traces.attrs"] et = h5_object.make_custom_group("extracellular_traces", path = "/acquisition/timeseries", \ attrs = group_attrs) for k in dict.keys(): if len(k.split(".")) == len(string.split(".")) + 1 and \ not k.split(".")[-1] == "attrs": name = k.split(".")[-1] attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if name == "ephys_raw_data": et.set_custom_dataset(name, dict[k], attrs=attrs) else: try: et.set_dataset(name, dict[k], attrs=attrs) except: et.set_custom_dataset(name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def top_datasets(self, string, dict, project_dir): dict["verbosity"] = "none" if debug: util.print_dict_keys(dict) h5_object = nwb_file.open(**dict) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def processing_extracellular_units(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) if re.search("top_datasets", string): self.processing_extracellular_units_top_datasets( string, dict, project_dir) elif re.search("EventWaveform", string): self.processing_extracellular_units_EventWaveform( string, dict, project_dir) elif re.search("UnitTimes", string): self.processing_extracellular_units_UnitTimes( string, dict, project_dir)
def processing_extracellular_units_UnitTimes(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) module_name = "extracellular_units" module_attrs = {"ancestry": "<Module>"} if "processing.extracellular_units.attrs" in dict.keys(): module_attrs = dict["processing.extracellular_units.attrs"] module_attrs["ancestry"] = "<Module>" mod = h5_object.make_group("<Module>", module_name, attrs=module_attrs) group_attrs = {} if "processing.extracellular_units.UnitTimes.attrs" in dict.keys(): group_attrs = dict[ "processing.extracellular_units.UnitTimes.attrs"] spk_times_iface = mod.make_group("UnitTimes", attrs=group_attrs) spk = {} for k in dict.keys(): data_name = k.split(".")[-1] if not data_name == "attrs": attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if len(k.split(".")) == 4: # Creating dataset cell_types, etc try: spk_times_iface.set_dataset(data_name, dict[k], attrs=attrs) except: spk_times_iface.set_custom_dataset(data_name, dict[k], attrs=attrs) elif len(k.split(".")) == 5: unit = k.split(".")[-2] attrs["ancestry"] = np.array(["<unit_N>"]) spk[unit] = spk_times_iface.make_group("<unit_N>", unit, \ attrs=attrs,abort = False) try: spk[unit].set_dataset(data_name, dict[k], attrs=attrs) except: spk[unit].set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def epochs(self, string, dict, project_dir): # string = epochs # no custom datasets are allowed if debug: util.print_dict_keys(dict) epoch_datasets = ["start_time", "stop_time", "tags", "units_present", \ "description", "ROIs", "ROI_planes"] h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) eg = h5_object.make_group("epochs", abort=False) processed_trials = [] tg = {} for k in dict.keys(): if len(k.split(".")) == 2: name = k.split(".")[-1] attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not re.search("rial", name): print " Warning: creating custom dataset epochs." + name try: eg.set_dataset(name, dict[k], attrs=attrs) except: eg.set_custom_dataset(name, dict[k], attrs=attrs) else: trial = k.split(".")[-2] if trial not in processed_trials: processed_trials.append(trial) tg[trial] = h5_object.make_group("<epoch_X>", trial, attrs = \ {"description" : "Data that belong to " + trial,\ "ancestry" : np.array(["epoch_X"])}) if len(k.split(".")) == 3 and not k.split(".")[-1] == "attrs": name = k.split(".")[-1] attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] try: tg[trial].set_dataset(name, dict[k], attrs=attrs) except: tg[trial].set_custom_dataset(name, dict[k], attrs=attrs) if not name in epoch_datasets: print " Warning: creating custom dataset epochs." + trial + "." + name if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def general_optophysiology(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) gg = h5_object.make_group("general", abort=False) op = gg.make_group("optophysiology", abort=False) for k in dict.keys(): data_name = k.split(".")[-1] if not data_name == "attrs": attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not re.search("fov_", k) and len(k.split(".")) == 3: # Creating description and custom datasets in general/optophysiology try: op.set_dataset(data_name, dict[k], attrs=attrs) except: op.set_custom_dataset(data_name, dict[k], attrs=attrs) elif re.search("fov_", k): group_name = k.split(".")[2] img_plane_group = op.make_group("<imaging_plane_X>", group_name, \ attrs = {"ancestry" : np.array(["imaging_plane_X"])}, abort=False) if not re.search("channel_", k): try: img_plane_group.set_dataset(data_name, dict[k], attrs=attrs) except: img_plane_group.set_custom_dataset(data_name, dict[k], attrs=attrs) else: name_channel = k.split(".")[3] channel_group = img_plane_group.make_group("<channel_X>", name_channel, \ attrs = {"ancestry" : np.array(["channel_X"])}, abort=False) try: channel_group.set_dataset(data_name, dict[k], attrs=attrs) except: channel_group.set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def processing_ROIs(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) h5_object = self.processing_ROIs_DfOverF(h5_object, string + ".DfOverF", dict, project_dir) dict2 = {} for k in dict.keys(): if re.search("ImageSegmentation", k): dict2[k] = dict[k] h5_object = self.processing_ROIs_ImageSegmentation(h5_object, string + ".ImageSegmentation", \ dict2, project_dir) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def general_optogenetics(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) optogenetics_datasets = [ "device", "excitation_lambda", "location", "stimulation_method" ] h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) gg = h5_object.make_group("general", abort=False) og = gg.make_group("optogenetics", abort=False) for k in dict.keys(): data_name = k.split(".")[-1] if len(k.split(".")) == 3 and not data_name == "attrs": # Creating custom datasets in group general/optogenetics attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not data_name in optogenetics_datasets: print " Warning: creating custom dataset general.optogenetics." + data_name try: og.set_dataset(data_name, dict[k], attrs=attrs) except: og.set_custom_dataset(data_name, dict[k], attrs=attrs) if k.split(".")[-3] == "optogenetics": osg = og.make_group("<site_X>", name=k.split(".")[-2], \ attrs={"ancestry" : np.array(["site_X"])}, abort=False) if len(k.split(".")) == 4 and not data_name == "attrs": # Creating datasets in group general/optogenetics/site_# attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not data_name in optogenetics_datasets: print " Warning: creating custom dataset general.optogenetics." \ + k.split(".")[-2] + "." + data_name try: osg.set_dataset(data_name, dict[k], attrs=attrs) except: osg.set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def general_devices(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) for k in dict.keys(): name = k.split(".")[-1] if not name == "attrs": attrs = attrs = {"ancestry": np.array(["device_X"])} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] attrs["ancestry"] = np.array(["device_X"]) h5_object.set_dataset("<device_X>", dict[k], name=name, \ attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def general_extracellular_ephys(self, string, dict, project_dir): # string = general.extracellular_ephys if debug: util.print_dict_keys(dict) extracellular_ephys_datasets = ["ADunit", "electrode_group", "electrode_map", \ "filtering"] shank_datasets = ["description", "device", "location"] h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) gg = h5_object.make_group("general", abort=False) eeg = gg.make_group("extracellular_ephys", abort=False) for k in dict.keys(): if k.split(".")[-2] == "extracellular_ephys": data_name = k.split(".")[-1] if not data_name == "attrs": attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not data_name in extracellular_ephys_datasets: print " Warning: creating custom dataset general.extracellular_ephys." + data_name try: eeg.set_dataset(data_name, dict[k], attrs=attrs) except: eeg.set_custom_dataset(data_name, dict[k], attrs=attrs) elif re.search("shank", k.split(".")[-2]): sg = eeg.make_group("<electrode_group_X>", k.split(".")[-2], \ attrs={"ancestry" : np.array(["electrode_group_X"])}, abort=False) data_name = k.split(".")[-1] if not data_name == "attrs": attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not data_name in shank_datasets: print " Warning: creating custom dataset general.extracellular_ephys.shank." + data_name try: sg.set_dataset(data_name, dict[k], attrs=attrs) except: sg.set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def processing_extracellular_units_EventWaveform(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) module_name = "extracellular_units" module_attrs = {"ancestry": np.array(["Module"])} if "processing.extracellular_units.attrs" in dict.keys(): module_attrs = dict["processing.extracellular_units.attrs"] module_attrs["ancestry"] = "extracellular_units" mod = h5_object.make_group("<Module>", module_name, attrs=module_attrs) group_attrs = {} if "processing.extracellular_units.EventWaveform.attrs" in dict.keys(): group_attrs = dict[ "processing.extracellular_units.EventWaveform.attrs"] spk_waves_iface = mod.make_group("EventWaveform") spk_waves_iface.set_attr("source", "EventWaveform in this module") spk = {} for k in dict.keys(): data_name = k.split(".")[-1] if not data_name == "attrs": attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if len(k.split(".")) == 5: unit = k.split(".")[-2] group_attrs = {"source": "---"} spk[unit] = spk_waves_iface.make_group("<SpikeEventSeries>", unit, \ attrs = group_attrs, abort = False) try: spk[unit] = spk[unit].set_dataset(data_name, dict[k], attrs=attrs) except: spk[unit] = spk[unit].set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def processing_timeseries(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) module_name = string.split(".")[1] series_name = string.split(".")[-1] if string + ".attrs" in dict.keys() and \ "series_type" in dict[string + ".attrs"].keys(): series_type = dict[string + ".attrs"]["series_type"] else: series_type = "<TimeSeries>" h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) try: module_attrs = dict["processing." + module_name + ".attrs"] except: module_attrs = {} module_attrs["ancestry"] = np.array(["Module"]) mod = h5_object.make_group("<Module>", module_name, attrs=module_attrs) iface = mod.make_group(string.split(".")[2], attrs=dict[string + ".attrs"]) tsg = iface.make_group(series_type, series_name, attrs=dict[string + ".attrs"]) for k in dict.keys(): data_name = k.split(".")[-1] if not data_name == "attrs" and len(k.split(".")) == 5: attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] try: tsg.set_dataset(data_name, dict[k], attrs=attrs) except: tsg.set_custom_dataset(data_name, dict[k], attrs=attrs) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()
def acquisition_timeseries(self, string, dict, project_dir): if debug: util.print_dict_keys(dict) h5_object = self.initialize_h5_object( os.path.join(project_dir, string + ".h5"), dict) if len(string.split(".")) == 3: # behavioral time series # string = acquisition.timeseries.<data_name> series_name = string.split(".")[-1] if string + ".attrs" in dict.keys() and \ "series_type" in dict[string + ".attrs"].keys(): series_type = dict[string + ".attrs"]["series_type"] else: series_type = '<TimeSeries>' series_path = "/acquisition/timeseries" attrs = {"series_type": "<TimeSeries>"} if string + ".attrs" in dict.keys(): attrs = dict[string + ".attrs"] series_type = '<TimeSeries>' tsg = h5_object.make_group(series_type, series_name, path=series_path, attrs=attrs) for k in dict.keys(): if len(k.split(".")) == len(string.split(".")) + 1: name = k.split(".")[-1] attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] if not name == "attrs": try: tsg.set_dataset(name, dict[k], attrs=attrs) except: tsg.set_custom_dataset(name, dict[k], attrs=attrs) elif len(string.split(".")) == 2: # image time series - ophys data only # string = acquisition.timeseries img_planes = [] for k in dict.keys(): if k.split(".")[-1] == "imaging_plane": img_planes.append(k.split(".")[2]) for img_plane in img_planes: group_attrs = {} if string + "." + img_plane + ".attrs" in dict.keys(): group_attrs = dict[string + ".attrs"] twop = h5_object.make_group("<TwoPhotonSeries>", img_plane, \ path='/acquisition/timeseries', attrs=group_attrs) for k in dict.keys(): if k.split(".")[2] == img_plane and not k.split( ".")[-1] == "attrs": name = k.split(".")[-1] attrs = {} if k + ".attrs" in dict.keys(): attrs = dict[k + ".attrs"] try: twop.set_dataset(name, dict[k], attrs=attrs) except: try: twop.set_custom_dataset(name, dict[k], attrs=attrs) except: pass else: sys.exit("Unsupported path string " + string) if verbose: print "Creating partial NWB file: ", os.path.join( project_dir, string + ".h5") h5_object.close()