#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Thu Apr 29 10:00:29 2021 @author: guido """ from iblvideo import run_session from oneibl.one import ONE one = ONE() # session = one.search(task_protocol='_iblrig_NPH_tasks_trainingChoiceWorld') session = one.search(task_protocol='_iblrig_tasks_opto_biasedChoiceWorld') for eid in session: print(f'Processing session {eid}') status = run_session(eid, machine='guido', cams=['left'], one=one, frames=10000)
downsample_factor = 20 uclusters = np.unique(spikes['clusters']) cols = ['c', 'b', 'g', 'y', 'k', 'r', 'm'] cols_cat = (cols * int(len(uclusters) / len(cols) + 10))[:len(uclusters)] col_dict = dict(zip(uclusters, cols_cat)) # downsample z = spikes['clusters'][::downsample_factor] x = spikes['times'][::downsample_factor] y = spikes['depths'][::downsample_factor] cols_int = [col_dict[x] for x in z] plt.scatter(x, y, marker='o', s=0.01, c=cols_int) plt.ylabel('depth [um]') plt.xlabel('time [sec]') plt.title('downsample factor: %s' % downsample_factor) if __name__ == '__main__': one = ONE() eid = one.search(subject='ZM_2407', date='2019-11-05', number=3) D = one.load(eid[0], clobber=False, download_only=True) alf_path = Path(D.local_path[0]).parent spikes = alf.io.load_object(alf_path, 'spikes') scatter_raster(spikes)
class RegistrationClient: def __init__(self, one=None): self.one = one if not one: self.one = ONE() self.dtypes = self.one.alyx.rest('dataset-types', 'list') self.file_extensions = [ df['file_extension'] for df in self.one.alyx.rest('data-formats', 'list') ] def create_sessions(self, root_data_folder, dry=False): # session creation on the create me flags. flag_files = Path(root_data_folder).glob('**/create_me.flag') for flag_file in flag_files: if dry: print(flag_file) continue logger_.info('creating session for ' + str(flag_file.parent)) # providing a false flag stops the registration after session creation status_str = self.register_session(flag_file.parent, file_list=False) if status_str: logger_.error(status_str) flag_file.unlink() def register_sync(self, root_data_folder, dry=False): flag_files = Path(root_data_folder).glob('**/register_me.flag') for flag_file in flag_files: if dry: print(flag_file) continue file_list = flags.read_flag_file(flag_file) logger_.info('registering ' + str(flag_file.parent)) status_str = self.register_session(flag_file.parent, file_list=file_list) if status_str: error_message = str(flag_file.parent) + ' failed registration' error_message += '\n' + ' ' * 8 + status_str error_message += traceback.format_exc() logger_.error(error_message) err_file = flag_file.parent.joinpath('register_me.error') flag_file.replace(err_file) with open(err_file, 'w+') as f: f.write(error_message) continue flags.write_flag_file(flag_file.parent.joinpath('flatiron.flag'), file_list=file_list) flag_file.unlink() if flag_file.parent.joinpath('create_me.flag').exists(): flag_file.parent.joinpath('create_me.flag').unlink() logger_.info('registered' + '\n') @log2sessionfile def register_session(self, ses_path, file_list=True, repository_name=None): if isinstance(ses_path, str): ses_path = Path(ses_path) # read meta data from the rig for the session from the task settings file settings_json_file = [ f for f in ses_path.glob('**/_iblrig_taskSettings.raw*.json') ] if not settings_json_file: logger_.error( ['could not find _iblrig_taskSettings.raw.json. Abort.']) return else: settings_json_file = settings_json_file[0] md = _read_settings_json_compatibility_enforced(settings_json_file) # query alyx endpoints for subject, error if not found try: subject = self.one.alyx.rest( 'subjects?nickname=' + md['SUBJECT_NAME'], 'list')[0] except IndexError: return 'Subject: ' + md[ 'SUBJECT_NAME'] + " doesn't exist in Alyx. ABORT." # look for a session from the same subject, same number on the same day _, session = self.one.search(subjects=subject['nickname'], date_range=md['SESSION_DATE'], number=md['SESSION_NUMBER'], details=True) try: user = self.one.alyx.rest('users', 'read', md["PYBPOD_CREATOR"][0]) except Exception: return 'User: '******'t exist in Alyx. ABORT" username = user['username'] if user else subject['responsible_user'] # load the trials data to get information about session duration ses_data = raw.load_data(ses_path) ses_duration_secs = _get_session_duration(ses_path, ses_data) start_time = ibllib.time.isostr2date(md['SESSION_DATETIME']) end_time = start_time + datetime.timedelta(seconds=ses_duration_secs) # this is the generic relative path: subject/yyyy-mm-dd/NNN gen_rel_path = Path(subject['nickname'], md['SESSION_DATE'], '{0:03d}'.format(int(md['SESSION_NUMBER']))) # checks that the number of actual trials and labeled number of trials check out assert (len(ses_data) == ses_data[-1]['trial_num']) # task specific logic if 'habituationChoiceWorld' in md['PYBPOD_PROTOCOL']: n_correct_trials = 0 else: n_correct_trials = ses_data[-1]['ntrials_correct'] # if nothing found create a new session in Alyx if not session: ses_ = { 'subject': subject['nickname'], 'users': [username], 'location': md['PYBPOD_BOARD'], 'procedures': ['Behavior training/tasks'], 'lab': subject['lab'], # 'project': project['name'], 'type': 'Experiment', 'task_protocol': md['PYBPOD_PROTOCOL'] + md['IBLRIG_VERSION_TAG'], 'number': md['SESSION_NUMBER'], 'start_time': ibllib.time.date2isostr(start_time), 'end_time': ibllib.time.date2isostr(end_time), 'n_correct_trials': n_correct_trials, 'n_trials': ses_data[-1]['trial_num'], 'json': json.dumps(md, indent=1), } session = self.one.alyx.rest('sessions', 'create', data=ses_) if md['SUBJECT_WEIGHT']: wei_ = { 'subject': subject['nickname'], 'date_time': ibllib.time.date2isostr(start_time), 'weight': md['SUBJECT_WEIGHT'], 'user': username } self.one.alyx.rest('weighings', 'create', wei_) else: session = session[0] logger_.info(session['url'] + ' ') # create associated water administration if not found if not session['wateradmin_session_related']: wa_ = { 'subject': subject['nickname'], 'date_time': ibllib.time.date2isostr(end_time), 'water_administered': ses_data[-1]['water_delivered'] / 1000, 'water_type': md['REWARD_TYPE'], 'user': username, 'session': session['url'][-36:], 'adlib': False } self.one.alyx.rest('water-administrations', 'create', wa_) # at this point the session has been created. If create only, exit if not file_list: return # register all files that match the Alyx patterns, warn user when files are encountered rename_files_compatibility(ses_path, md['IBLRIG_VERSION_TAG']) F = { } # empty dict whose keys will be relative paths and content filenames for fn in ses_path.glob('**/*.*'): if fn.suffix in ['.flag', '.error', '.avi', '.log']: logger_.debug('Excluded: ', str(fn)) continue if not self._match_filename_dtypes(fn): logger_.warning('No matching dataset type for: ' + str(fn)) continue if fn.suffix not in self.file_extensions: logger_.warning( 'No matching dataformat (ie. file extension) for: ' + str(fn)) continue if not _register_bool(fn.name, file_list): logger_.debug('Not in filelist: ' + str(fn)) continue try: assert (str(gen_rel_path) in str(fn)) except AssertionError: strerr = 'ALF folder mismatch: data is in wrong subject/date/number folder. \n' strerr += ' Expected ' + str( gen_rel_path) + ' actual was ' + str(fn) return strerr # extract the relative path of the file rel_path = Path(str(fn)[str(fn).find(str(gen_rel_path)):]).parent if str(rel_path) not in F.keys(): F[str(rel_path)] = [fn.name] else: F[str(rel_path)].append(fn.name) logger_.info('Registering ' + str(fn)) for rpath in F: r_ = { 'created_by': username, 'path': rpath, 'filenames': F[rpath], } self.one.alyx.post('/register-file', data=r_) def _match_filename_dtypes(self, full_file): import re patterns = [ dt['filename_pattern'] for dt in self.dtypes if dt['filename_pattern'] ] for pat in patterns: reg = pat.replace('.', r'\.').replace('_', r'\_').replace('*', r'.+') if re.match(reg, Path(full_file).name, re.IGNORECASE): return True return False
## Init from oneibl.one import ONE from ibllib.misc import pprint one = ONE(base_url='https://test.alyx.internationalbrainlab.org', username='******', password='******') ## Find an experiment eid = one.search(users='olivier', date_range=['2018-08-24', '2018-08-24']) pprint(eid) one.search_terms() ## List dataset types for a session eid = 'cf264653-2deb-44cb-aa84-89b82507028a' one.list(eid) ## More Info about a session d = one.list(eid, 'All') ## Get More Info about datasets d = one.list(eid, details=True) print(d) print(d) ## List #1 one.list(None, 'dataset-types') one.list(None, 'users') one.list(None, 'subjects') ## Load #1 dataset_types = [ 'clusters.templateWaveforms', 'clusters.probes', 'clusters.depths'
if __name__ == '__main__': BIN_SIZE = 0.025 # seconds SMOOTH_SIZE = 0.025 # seconds; standard deviation of gaussian kernel PRE_TIME = 0.25 # seconds to plot before event onset POST_TIME = 1.0 # seconds to plot after event onset RESULTS_DIR = '/datadisk/scratch' # get the data from flatiron subject = 'KS004' date = '2019-09-25' number = 1 one = ONE() eid = one.search(subject=subject, date=date, number=number, task_protocol='ephysChoiceWorld') session_info = one.load(eid[0], clobber=False, download_only=True) session_path = get_session_path(session_info) alf_path = os.path.join(session_path, 'alf') # load objects spikes = ioalf.load_object(alf_path, 'spikes') clusters = ioalf.load_object(alf_path, 'clusters') trials = ioalf.load_object(alf_path, '_ibl_trials') # containers to store results align_events = ['stimOn', 'stimOff', 'feedback'] cluster_ids = np.unique( spikes['clusters']) # define subset of clusters to plot peth_means = {
extent=np.r_[times[[0, -1]], Clusters[[0, -1]]]) plt.xlabel('Time (s)') plt.ylabel('Cluster #; ordered by depth') plt.show() # plt.savefig('/home/mic/Rasters/%s.svg' %(trial_number)) # plt.close('all') plt.tight_layout() if __name__ == '__main__': # get data one = ONE() eid = one.search(lab='wittenlab', date='2019-08-04') D = one.load(eid[0], clobber=False, download_only=True) alf_path = Path(D.local_path[0]).parent spikes = alf.io.load_object(alf_path, 'spikes') # bin activity T_BIN = 0.01 # [sec] R, times, Clusters = bincount2D(spikes['times'], spikes['clusters'], T_BIN) # Order activity by anatomical depth of neurons d = dict(zip(spikes['clusters'], spikes['depths'])) y = sorted([[i, d[i]] for i in d]) isort = np.argsort([x[1] for x in y]) R = R[isort, :] # Check the number of clusters x number of time bins
import matplotlib.pyplot as plt from ibllib.io import raw_data_loaders import alf.io from oneibl.one import ONE one = ONE() dataset_types = [ 'trials.goCue_times', '_iblrig_taskData.raw', '_iblmic_audioSpectrogram.frequencies', '_iblmic_audioSpectrogram.power', '_iblmic_audioSpectrogram.times_mic' ] eIDs = one.search(task_protocol='bias', location='_iblrig_churchlandlab_ephys_0', dataset_types=dataset_types) # eIDs = '098bdac5-0e25-4f51-ae63-995be7fe81c7' # TEST EXAMPLE is_plot = False for i_eIDs in range(0, len(eIDs)): one.load(eIDs[i_eIDs], dataset_types=dataset_types, download_only=True) session_path = one.path_from_eid(eIDs[i_eIDs]) c = raw_data_loaders.load_data(session_path) n_trial = len(c) # -- Get spectrogram TF = alf.io.load_object(session_path.joinpath('raw_behavior_data'), 'audioSpectrogram', namespace='iblmic')
Get list of subjects associated to the certification recording project. TODO not finished: number of subject per lab, number of recording per lab ''' # Author: Gaelle Chapuis import numpy from oneibl.one import ONE one = ONE() # base_url='https://dev.alyx.internationalbrainlab.org' dataset_types = ['spikes.times', 'spikes.clusters'] # eid1, det1 = one.search(project='ibl_certif_neuropix_recording', # dataset_types=dataset_types, details=True) eid, det = one.search(task_protocol='ephys_certification', dataset_types=dataset_types, details=True) sub = [p['subject'] for p in det] # sub_unique = list(set(sub)) lab = [p['lab'] for p in det] # lab_unique = list(set(lab)) # task = [p['task_protocol'] for p in det] # task_unique = list(set(task)) # -- How many animals were used per lab su, ind_su = numpy.unique(sub, return_index=True) lab_arr = numpy.array(lab) lu = lab_arr[ind_su]
import numpy as np from brainbox.population import decode from sklearn.utils import shuffle from oneibl.one import ONE import brainbox.io.one as bbone # %% Load in data one = ONE() eid = one.search(subject='ZM_2240', date_range=['2020-01-23', '2020-01-23']) spikes, clusters = bbone.load_spike_sorting(eid[0], one=one) trials = one.load_object(eid[0], 'trials') # %% Only use units with KS2 label 'good' from probe00 spikes = spikes['probe00'] clusters = clusters['probe00'] clusters_to_use = clusters.metrics.ks2_label == 'good' spikes.times = spikes.times[np.isin( spikes.clusters, clusters.metrics.cluster_id[clusters_to_use])] spikes.clusters = spikes.clusters[np.isin( spikes.clusters, clusters.metrics.cluster_id[clusters_to_use])] cluster_ids = clusters.metrics.cluster_id[clusters_to_use] # %% Do decoding print('Decoding whether the stimulus was on the left or the right..') stim_times = trials.goCue_times stim_sides = np.isnan(trials.contrastLeft).astype(int) # Decode left vs right stimulus from a 1 second window after stimulus onset using default settings:
def xcorr(x, y, maxlags=None): """Cross-correlation between two 1D signals of the same length.""" ns = len(x) if len(y) != ns: raise ValueError("x and y should have the same length.") maxlags = maxlags or ns - 1 return np.correlate(x, y, mode='full')[ns - 1 - maxlags:ns + maxlags] T_BIN = 0.01 # seconds CORR_LEN = 1 # seconds CORR_BINS = int(CORR_LEN / T_BIN) # bins # get the data from flatiron and the current folder one = ONE() eid = one.search(subject='ZM_1150', date='2019-05-07', number=1) D = one.load(eid[0], clobber=False, download_only=True) session_path = Path(D.local_path[0]).parent # load objects spikes = ioalf.load_object(session_path, 'spikes') # Get a Bunch instance. b = one_to_bunch(spikes) # Compute the firing rates. rates = firing_rates(b.spike_times, b.spike_clusters, T_BIN) # Note: I would rather just use spikes['times'] and spikes['clusters'] instead of going # via a Bunch or DataFrame or similar... # Compute the cross-correlation between the firing rate of two neurons.
def launch_phy(probe_name, eid=None, subj=None, date=None, sess_no=None, one=None): """ Launch phy given an eid and probe name. TODO calculate metrics and save as .tsvs to include in GUI when launching? """ # This is a first draft, no error handling and a draft dataset list. # Load data from probe # # -------------------- # if one is None: one = ONE() dtypes = [ 'spikes.times', 'spikes.clusters', 'spikes.amps', 'spikes.templates', 'spikes.samples', 'spikes.depths', 'templates.waveforms', 'templates.waveformsChannels', 'clusters.uuids', 'clusters.metrics', 'clusters.waveforms', 'clusters.waveformsChannels', 'clusters.depths', 'clusters.amps', 'clusters.channels', 'channels.probes', 'channels.rawInd', 'channels.localCoordinates', # 'ephysData.raw.ap' '_phy_spikes_subset.waveforms', '_phy_spikes_subset.spikes', '_phy_spikes_subset.channels' ] if eid is None: eid = one.search(subject=subj, date=date, number=sess_no)[0] _ = one.load(eid, dataset_types=dtypes, download_only=True) ses_path = one.path_from_eid(eid) alf_probe_dir = os.path.join(ses_path, 'alf', probe_name) ephys_file_dir = os.path.join(ses_path, 'raw_ephys_data', probe_name) raw_files = glob.glob(os.path.join(ephys_file_dir, '*ap.*bin')) raw_file = [raw_files[0]] if raw_files else None # TODO download ephys meta-data, and extract TemplateController input arg params # Launch phy # # -------------------- # add_default_handler('DEBUG', logging.getLogger("phy")) add_default_handler('DEBUG', logging.getLogger("phylib")) create_app() controller = TemplateController(dat_path=raw_file, dir_path=alf_probe_dir, dtype=np.int16, n_channels_dat=384, sample_rate=3e4, plugins=['IBLMetricsPlugin'], plugin_dirs=[Path(__file__).resolve().parent]) gui = controller.create_gui() gui.show() run_app() gui.close() controller.model.close()
help='Date of session YYYY-MM-DD') parser.add_argument('-n', '--session_no', default=False, required=True, help='Session Number') parser.add_argument('-e', '--ephys_data', default=False, required=False, help='Set True to load ephys.bin data') args = parser.parse_args() one = ONE() eid = one.search(subject=str(args.subject), date=str(args.date), number=args.session_no)[0] data_path = one.load(eid, clobber=False, download_only=True) print( f'Successfully loaded data for {args.subject} {args.date} session no. {args.session_no}' ) if args.ephys_data: print('Loading ephys.bin data') ephys_types = [ 'ephysData.raw.ch', 'ephysData.raw.meta', 'ephysData.raw.ap', 'ephysData.raw.lf' ] ephys_path = one.load(eid, dataset_types=ephys_types, clobber=False,
t0, t1 = intervals[j, :] # Count the number of spikes in the window, for each neuron. x = np.bincount(spike_clusters[intervals_idx[j, 0]:intervals_idx[j, 1]], minlength=cluster_ids.max() + 1) counts[:, j] = x[cluster_ids] return counts, cluster_ids # %% # Set path to save plots DATA_PATH, FIG_PATH, _ = paths() FIG_PATH = join(FIG_PATH, 'TimeConstant') # Load in data eids = one.search(subject=SUBJECT, date_range=DATE) spikes, clusters, channels = bbone.load_spike_sorting_with_channel(eids[0], one=one) trials = one.load_object(eids[0], 'trials') # Only use single units probe = 'probe' + PROBE spikes[probe].times = spikes[probe].times[np.isin( spikes[probe].clusters, clusters[probe].metrics.cluster_id[ clusters[probe].metrics.ks2_label == 'good'])] spikes[probe].clusters = spikes[probe].clusters[np.isin( spikes[probe].clusters, clusters[probe].metrics.cluster_id[ clusters[probe].metrics.ks2_label == 'good'])] # Convert into seconds BIN_SIZE_S = BIN_SIZE / 1000
:return: x and y numpy arrays """ x = np.empty(0) y = np.empty(0) for key in list(dlc_dict.keys()): if key[-1] == 'x': x = np.append(x, dlc_dict[key][frame_number]) if key[-1] == 'y': y = np.append(y, dlc_dict[key][frame_number]) return x, y # Query sessions with available DLC data using ONE one = ONE() dtypes = ['camera.dlc', 'camera.times'] eids, ses_info = one.search(dataset_types=dtypes, details=True) # Loop over sessions FRAME_NR = 5000 for i, eid in enumerate(eids): if np.mod(i+1, 5) == 0: print('Running session %d of %d' % (i+1, len(eids))) # Load in data d = one.load(eid, dataset_types=dtypes, download_only=True, dclass_output=True) ses_path = Path(d.local_path[0]).parent dlc_dict = alf.io.load_object(ses_path, '_ibl_leftCamera', short_keys=True) video_path = Path(ses_path.parent, 'raw_video_data', '_iblrig_leftCamera.raw.mp4') # Get DLC and video data if (np.isnan(dlc_dict['pupil_top_r_x'][0])) or (np.size(dlc_dict['pupil_top_r_x']) < 1000):
class Alyx2NWBMetadata: def __init__(self, eid=None, one_obj=None, **one_search_kwargs): """ Query the sessions, subject, lab tables of the Alyx database using the ONE api. Retrieve a mice experiment's metadata as well as the various data types (ONE format) created during the experiment like: Trials, Behavior, Electrophysiology(raw, spike sorted), Stimulus, Probes used. Parameters ---------- eid: str uuid of IBL experiment one_obj: ONE() one object created after user authenticated connection to ALyx servers one_search_kwargs: dict various search terms to retrieve an eid of interest using the ONE api to query Alyx. """ if one_obj is None: self.one_obj = ONE() elif not isinstance(one_obj, OneAbstract): raise Exception('one_obj is not of ONE class') else: self.one_obj = one_obj if eid is None: eid = self.one_obj.search(**one_search_kwargs) if len(eid) > 1: print(f'nos of EIDs found for your search query: {len(eid)}, ' f'generating metadata from the first') if input('continue? y/n') == 'y': pass else: exit() self.eid = eid[0] elif isinstance(eid, list): self.eid = eid[0] else: self.eid = eid self.one_search_kwargs = one_search_kwargs self.schema = nwb_schema self.dataset_description_list = self._get_dataset_details() self.eid_session_info = self._retrieve_eid_endpoint() self.dataset_type_list = self._list_eid_metadata('dataset_type') self.users_list = self._list_eid_metadata('users') self.subjects_list = self._list_eid_metadata('subjects') self.labs_list = self._list_eid_metadata('labs') self.dataset_details, self.dataset_simple = self._dataset_type_parse() self._get_lab_table() self._get_subject_table() def _get_datetime(self, dtstr, format='%Y-%m-%dT%X'): if '.' in dtstr: dtstr = dtstr.split('.')[0] if len(dtstr) > 19: dtstr = dtstr[:19] if len(dtstr) == 10: format = '%Y-%m-%d' elif len(dtstr) == 19: if 'T' in dtstr: format = '%Y-%m-%dT%X' else: format = '%Y-%m-%d %X' try: return datetime.strptime(dtstr, format) except: raise Exception('could not convert to datetime') def _get_dataset_details(self): """ Retrieves all datasets in the alyx database currently. Retrieves a list of dicts with keys like id, name, created_by,description etc. Uses only name and description. Returns ------- list List of dicts: {<dataset-name> : <dataset_description> """ data_url_resp = self.one_obj.alyx.rest('dataset-types', 'list') return {i['name']: i['description'] for i in data_url_resp} def _list_eid_metadata(self, list_type): """ Uses one's list method to get the types of <list_type> data from the given eid. Parameters ---------- list_type: str one of strings from >>> ONE().search_terms() Returns ------- list """ return self.one_obj.list(self.eid, list_type) def _retrieve_eid_endpoint(self): """ To get the current sessions url response. Contains all the session metadata as well as the current datasets etc. Returns ------- list list of server responses. """ return self.one_obj.alyx.rest('sessions/' + self.eid, 'list') def _get_lab_table(self): self.lab_table = self.one_obj.alyx.rest('labs', 'list') def _get_subject_table(self): self.subject_table = self.one_obj.alyx.rest( 'subjects/' + self.eid_session_info['subject'], 'list') def _dataset_type_parse(self): """ Returns ------- list list of dicts: {<dataset object name>: (eg. spikes, clusters etc) [ {name: objects attribute type (eg. times, intervals etc description: attributes description} {name: objects attribute type (eg. times, intervals etc description: attributes description} ] } """ split_list_objects = [i.split('.')[0] for i in self.dataset_type_list] split_list_attributes = [ '.'.join(i.split('.')[1:]) for i in self.dataset_type_list ] dataset_description = [ self.dataset_description_list[i] for i in self.dataset_type_list ] split_list_objects_dict_details = dict() split_list_objects_dict = dict() for obj in set(split_list_objects): split_list_objects_dict_details[obj] = [] split_list_objects_dict[obj] = [] for att_idx, attrs in enumerate(split_list_attributes): append_dict = { 'name': attrs, 'description': dataset_description[att_idx] } # 'extension': dataset_extension[att_idx] } split_list_objects_dict_details[ split_list_objects[att_idx]].extend([append_dict]) split_list_objects_dict[split_list_objects[att_idx]].extend( [attrs]) dataset_type_list = split_list_objects_dict_details dataset_type_list_simple = split_list_objects_dict return dataset_type_list, dataset_type_list_simple @staticmethod def _unpack_dataset_details(dataset_details, object_name, custom_attrs=None, match_str=' '): """ Unpacks the dataset_details into: Parameters ---------- dataset_details: dict self.dataset_details object_name: str eg: spikes, clusters, Ecephys custom_attrs: list attrs to unpack match_str: regex match string: attrs to exclude (like .times/.intervals etc) Returns ------- datafiles: str ex: 'face.motionEnergy' datanames: str ex: 'motionEnergy' datadesc: str ex: <description string for motionEnergy> """ cond = lambda x: re.match(match_str, x) datafiles_all = [ object_name + '.' + ii['name'] for ii in dataset_details[object_name] if not cond(ii['name']) ] datafiles_names_all = [ ii['name'] for ii in dataset_details[object_name] if not cond(ii['name']) ] datafiles_desc_all = [ ii['description'] for ii in dataset_details[object_name] if not cond(ii['name']) ] if custom_attrs: datafiles_inc = [] datafiles_names_inc = [] datafiles_desc_inc = [] for attrs in custom_attrs: datafiles_inc.extend([ i for i in datafiles_all if i in object_name + '.' + attrs ]) datafiles_names_inc.extend([ datafiles_names_all[j] for j, i in enumerate(datafiles_all) if i in object_name + '.' + attrs ]) datafiles_desc_inc.extend([ datafiles_desc_all[j] for j, i in enumerate(datafiles_all) if i in object_name + '.' + attrs ]) else: datafiles_inc = datafiles_all datafiles_names_inc = datafiles_names_all datafiles_desc_inc = datafiles_desc_all return datafiles_inc, datafiles_names_inc, datafiles_desc_inc def _initialize_container_dict(self, name=None, default_value=None): if default_value is None: default_value = dict() if name: return dict({name: default_value.copy()}) else: return None def _get_all_object_names(self): return sorted( list(set([i.split('.')[0] for i in self.dataset_type_list]))) def _get_current_object_names(self, obj_list): loop_list = [] for j, k in enumerate(obj_list): loop_list.extend( [i for i in self._get_all_object_names() if k == i]) return loop_list def _get_timeseries_object(self, dataset_details, object_name, ts_name, custom_attrs=None, drop_attrs=None, **kwargs): """ Parameters ---------- dataset_details: dict self.dataset_details object_name: str name of hte object_name in the IBL datatype ts_name: str the key name for the timeseries list custom_attrs: list Attributes to consider drop_attrs: list Attributes to drop kwargs additional keys/values to add to the default timeseries. For derivatives of TimeSEries Returns ------- dict() { "time_series": [ { "name": "face_motionEnergy", "data": "face.motionEnergy", "timestamps": "face.timestamps", "description": "Features extracted from the video of the frontal aspect of the subject, including the subject\\'s face and forearms." }, { "name": "_ibl_lickPiezo_times", "data": "_ibl_lickPiezo.raw", "timestamps": "_ibl_lickPiezo.timestamps", "description": "Voltage values from a thin-film piezo connected to the lick spout, so that values are proportional to deflection of the spout and licks can be detected as peaks of the signal." } ] } """ matchstr = r'.*time.*|.*interval.*' timeattr_name = [ i['name'] for i in dataset_details[object_name] if re.match(matchstr, i['name']) ] dataset_details[object_name], _ = self._drop_attrs( dataset_details[object_name].copy(), drop_attrs) datafiles, datafiles_names, datafiles_desc = \ self._unpack_dataset_details(dataset_details.copy(), object_name, custom_attrs, match_str=matchstr) if timeattr_name: datafiles_timedata, datafiles_time_name, datafiles_time_desc = \ self._unpack_dataset_details(dataset_details.copy(), object_name, timeattr_name) elif not kwargs: # if no timestamps info, then let this fields be data return {ts_name: []} else: datafiles_timedata, datafiles_time_name, datafiles_time_desc = \ datafiles, datafiles_names, datafiles_desc if not datafiles: if not kwargs: return {ts_name: []} # datafiles_names = datafiles_time_name # datafiles_desc = datafiles_time_desc # datafiles = ['None'] timeseries_dict = {ts_name: [None] * len(datafiles)} for i, j in enumerate(datafiles): original = { 'name': datafiles_names[i], 'description': datafiles_desc[i], 'timestamps': datafiles_timedata[0], 'data': datafiles[i] } original.update(**kwargs) timeseries_dict[ts_name][i] = { k: v for k, v in original.items() if v is not None } return timeseries_dict @staticmethod def _attrnames_align(attrs_dict, custom_names): """ the attributes that receive the custom names are reordered to be first in the list Parameters. This assigns description:'no_description' to those that are not found. This will later be used(nwb_converter) as an identifier for non-existent data for the given eid. ---------- attrs_dict:list list of dict(attr_name:'',attr_description:'') custom_names same as 'default_colnames_dict' in self._get_dynamictable_object Returns ------- dict() """ attrs_list = [i['name'] for i in attrs_dict] list_id_func_exclude = \ lambda val, comp_list, comp_bool: [i for i, j in enumerate(comp_list) if comp_bool & (j == val)] cleanup = lambda x: [i[0] for i in x if i] if custom_names: custom_names_list = [i for i in list(custom_names.values())] custom_names_dict = [] for i in range(len(custom_names_list)): custom_names_dict.extend([{ 'name': custom_names_list[i], 'description': 'no_description' }]) attr_list_include_idx = cleanup([ list_id_func_exclude(i, attrs_list, True) for i in custom_names_list ]) attr_list_exclude_idx = set(range(len(attrs_list))).difference( set(attr_list_include_idx)) custom_names_list_include_idx = [ i for i, j in enumerate(custom_names_list) if list_id_func_exclude(j, attrs_list, True) ] for ii, jj in enumerate(custom_names_list_include_idx): custom_names_dict[custom_names_list_include_idx[ ii]] = attrs_dict[attr_list_include_idx[ii]] custom_names_list[custom_names_list_include_idx[ ii]] = attrs_list[attr_list_include_idx[ii]] extend_dict = [attrs_dict[i] for i in attr_list_exclude_idx] extend_list = [attrs_list[i] for i in attr_list_exclude_idx] custom_names_dict.extend(extend_dict) custom_names_list.extend(extend_list) return custom_names_dict, custom_names_list else: out_dict = attrs_dict out_list = attrs_list return out_dict, out_list @staticmethod def _drop_attrs(dataset_details, drop_attrs, default_colnames_dict=None): """ Used to remove given attributes of the IBL dataset. Parameters ---------- dataset_details: list self.dataset_details['clusters'] [ { 'name': 'amps', 'description': description }, { 'name': 'channels', 'description': description } ] drop_attrs: list list of str: attribute names to drop of the self.dataset_details dict default_colnames_dict Returns ------- dataset_details: list list without dictionaries with 'name' as in drop_attrs """ attrs_list = [i['name'] for i in dataset_details] if default_colnames_dict is not None: default_colnames_dict_copy = default_colnames_dict.copy() for i, j in default_colnames_dict.items(): if j not in attrs_list: default_colnames_dict_copy.pop(i) else: default_colnames_dict_copy = default_colnames_dict if drop_attrs is None: return dataset_details, default_colnames_dict_copy elif default_colnames_dict is not None: for i, j in default_colnames_dict.items(): if j in drop_attrs and j in attrs_list: default_colnames_dict_copy.pop(i) dataset_details_return = [ dataset_details[i] for i, j in enumerate(attrs_list) if j not in drop_attrs ] return dataset_details_return, default_colnames_dict_copy @staticmethod def _get_dynamictable_array(**kwargs): """ Helper to dynamictable object method Parameters ---------- kwargs keys and values that define the dictionary, both keys and values are lists where each index would slice all the keys/values and create a dict out of that Returns ------- list list of dictionaries each with the keys and values from kwargs """ custom_keys = list(kwargs.keys()) custom_data = list(kwargs.values()) out_list = [None] * len(custom_data[0]) for ii, jj in enumerate(custom_data[0]): out_list[ii] = dict().copy() for i, j in enumerate(custom_keys): out_list[ii][j] = custom_data[i][ii] return out_list def _get_dynamictable_object(self, dataset_details, object_name, dt_name, default_colnames_dict=None, custom_attrs=None, drop_attrs=None): """ Parameters ---------- dataset_details self.dataset_details for each eid object_name:str object from the IBL data types from which to create this table. dt_name:str custom name for the dynamic table. Its the key with the value being dynamictable_array default_colnames_dict:dict() keys are the custom names of the columns, corresponding values are the attributes which have to be renamed. custom_attrs:list list of attributes for the given IBL object in object_name to be considered, all others are ignored Returns ------- outdict:dict() example output below: {'Trials': [ { "name": "column1 name", "data": "column data uri (string)", "description": "col1 description" }, { "name": "column2 name", "data": "column data uri (string)", "description": "col2 description" } ] } """ dataset_details[object_name], default_colnames_dict = self._drop_attrs( dataset_details[object_name].copy(), drop_attrs, default_colnames_dict) dataset_details[object_name], _ = self._attrnames_align( dataset_details[object_name].copy(), default_colnames_dict) if not default_colnames_dict: default_colnames = [] else: default_colnames = list(default_colnames_dict.keys()) custom_columns_datafilename, custom_columns_name, custom_columns_description = \ self._unpack_dataset_details(dataset_details.copy(), object_name, custom_attrs) custom_columns_name[:len(default_colnames)] = default_colnames in_list = self._get_dynamictable_array( name=custom_columns_name, data=custom_columns_datafilename, description=custom_columns_description) outdict = {dt_name: in_list} return outdict @property def eid_metadata(self): return dict(eid=self.eid) @property def probe_metadata(self): probes_metadata_dict = self._initialize_container_dict( 'Probes', default_value=[]) probe_list = self.eid_session_info['probe_insertion'] probe_dict_keys = ['id', 'model', 'name', 'trajectory_estimate'] input_dict = dict() for key in probe_dict_keys: if key == 'trajectory_estimate': input_dict.update({ key: [[json.dumps(l) for l in probe_list[i].get(key, ["None"])] for i in range(len(probe_list))] }) else: input_dict.update({ key: [ probe_list[i].get(key, "None") for i in range(len(probe_list)) ] }) probes_metadata_dict['Probes'].extend( self._get_dynamictable_array(**input_dict)) return probes_metadata_dict @property def nwbfile_metadata(self): nwbfile_metadata_dict = self._initialize_container_dict('NWBFile') nwbfile_metadata_dict['NWBFile'].update( session_start_time=self._get_datetime( self.eid_session_info['start_time']), keywords=[ ','.join(self.eid_session_info['users']), self.eid_session_info['lab'], 'IBL' ], experiment_description=self.eid_session_info['project'], session_id=self.eid, experimenter=self.eid_session_info['users'], identifier=self.eid, institution=[ i['institution'] for i in self.lab_table if i['name'] == [self.eid_session_info['lab']][0] ][0], lab=self.eid_session_info['lab'], protocol=self.eid_session_info['task_protocol'], surgery='none', notes=', '.join([ f"User:{i['user']}{i['text']}" for i in self.eid_session_info['notes'] ]), session_description=','.join(self.eid_session_info['procedures'])) return nwbfile_metadata_dict @property def sessions_metadata(self): sessions_metadata_dict = self._initialize_container_dict( 'IBLSessionsData') custom_fields = [ 'location', 'project', 'type', 'number', 'end_time', 'parent_session', 'url', 'qc' ] sessions_metadata_dict['IBLSessionsData'] = { i: str(self.eid_session_info[i]) if i not in ['procedures', 'number'] else self.eid_session_info[i] for i in custom_fields } sessions_metadata_dict['IBLSessionsData']['extended_qc'] = json.dumps( self.eid_session_info['extended_qc']) sessions_metadata_dict['IBLSessionsData']['json'] = json.dumps( self.eid_session_info['json']) sessions_metadata_dict['IBLSessionsData']['wateradmin_session_related'] = \ [json.dumps(i) for i in self.eid_session_info['wateradmin_session_related']]\ if len(self.eid_session_info['wateradmin_session_related']) > 0 else ['None'] sessions_metadata_dict['IBLSessionsData']['notes'] = \ [json.dumps(i) for i in self.eid_session_info['notes']]\ if len(self.eid_session_info['notes']) > 0 else ['None'] return sessions_metadata_dict @property def subject_metadata(self): subject_metadata_dict = self._initialize_container_dict('IBLSubject') sub_table_dict = deepcopy(self.subject_table) if sub_table_dict: subject_metadata_dict['IBLSubject'] = dict( age=f'P{sub_table_dict.pop("age_weeks")}W', subject_id=sub_table_dict.pop('id'), description=sub_table_dict.pop('description'), genotype=','.join(sub_table_dict.pop('genotype')), sex=sub_table_dict.pop('sex'), species=sub_table_dict.pop('species'), weight=str(sub_table_dict.pop('reference_weight')), date_of_birth=self._get_datetime( sub_table_dict.pop('birth_date')), **sub_table_dict) water_admin_data = [json.dumps(i) for i in subject_metadata_dict['IBLSubject']['water_administrations']]\ if len(subject_metadata_dict['IBLSubject']['water_administrations']) > 0 else ['None'] subject_metadata_dict['IBLSubject'].update( weighings=[ json.dumps(i) for i in subject_metadata_dict['IBLSubject']['weighings'] ], water_administrations=water_admin_data) temp_metadatadict = deepcopy(subject_metadata_dict['IBLSubject']) for key, val in temp_metadatadict.items(): if isinstance(val, list) and len(val) == 0: _ = subject_metadata_dict['IBLSubject'].pop(key) return subject_metadata_dict @property def surgery_metadata(self): # currently not exposed by api return dict() @property def behavior_metadata(self): behavior_metadata_dict = self._initialize_container_dict('Behavior') behavior_objects = [ 'wheel', 'wheelMoves', 'licks', 'lickPiezo', 'face', 'eye', 'camera' ] current_behavior_objects = self._get_current_object_names( behavior_objects) for object_name in current_behavior_objects: if 'wheel' == object_name: behavior_metadata_dict['Behavior']['BehavioralTimeSeries'] = \ self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series') if 'wheelMoves' in object_name: behavior_metadata_dict['Behavior']['BehavioralEpochs'] = \ self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_intervals') if 'lickPiezo' in object_name: behavior_metadata_dict['Behavior']['BehavioralTimeSeries'][ 'time_series'].extend( self._get_timeseries_object( self.dataset_details.copy(), object_name, 'time_series')['time_series']) if 'licks' in object_name: behavior_metadata_dict['Behavior']['BehavioralEvents'] = \ self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series') if 'face' in object_name: behavior_metadata_dict['Behavior']['BehavioralTimeSeries'][ 'time_series'].extend( self._get_timeseries_object( self.dataset_details.copy(), object_name, 'time_series')['time_series']) if 'eye' in object_name: behavior_metadata_dict['Behavior']['PupilTracking'] = \ self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series') if 'camera' in object_name: behavior_metadata_dict['Behavior']['Position'] = \ self._get_timeseries_object(self.dataset_details.copy(), object_name, 'spatial_series', name='camera_dlc') if len(behavior_metadata_dict['Behavior']['Position']['spatial_series']) > 0 and \ behavior_metadata_dict['Behavior']['Position']['spatial_series'][0]['data']== \ behavior_metadata_dict['Behavior']['Position']['spatial_series'][0]['timestamps']: behavior_metadata_dict['Behavior']['Position'][ 'spatial_series'][0][ 'timestamps'] = '_iblrig_Camera.timestamps' return behavior_metadata_dict @property def trials_metadata(self): trials_metadata_dict = self._initialize_container_dict('Trials') trials_objects = ['trials'] current_trial_objects = self._get_current_object_names(trials_objects) for object_name in current_trial_objects: if 'trial' in object_name: trials_metadata_dict = self._get_dynamictable_object( self.dataset_details.copy(), 'trials', 'Trials', default_colnames_dict=dict(start_time='intervals', stop_time='intervals')) return trials_metadata_dict @property def stimulus_metadata(self): stimulus_objects = [ 'sparseNoise', 'passiveBeeps', 'passiveValveClick', 'passiveVisual', 'passiveWhiteNoise' ] stimulus_metadata_dict = self._initialize_container_dict('Stimulus') current_stimulus_objects = self._get_current_object_names( stimulus_objects) for object_name in current_stimulus_objects: if 'sparseNoise' in object_name: stimulus_metadata_dict['Stimulus'] = \ self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series') if 'passiveBeeps' in object_name: stimulus_metadata_dict['Stimulus']['time_series'].extend( self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')['time_series']) if 'passiveValveClick' in object_name: stimulus_metadata_dict['Stimulus']['time_series'].extend( self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')['time_series']) if 'passiveVisual' in object_name: stimulus_metadata_dict['Stimulus']['time_series'].extend( self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')['time_series']) if 'passiveWhiteNoise' in object_name: stimulus_metadata_dict['Stimulus']['time_series'].extend( self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')['time_series']) return stimulus_metadata_dict @property def device_metadata(self): device_metadata_dict = self._initialize_container_dict( 'Device', default_value=[]) device_metadata_dict['Device'].extend( self._get_dynamictable_array(name=['NeuroPixels probe'], description=['NeuroPixels probe'])) return device_metadata_dict @property def units_metadata(self): units_objects = ['clusters', 'spikes'] metrics_columns = [ 'cluster_id', 'cluster_id.1', 'num_spikes', 'firing_rate', 'presence_ratio', 'presence_ratio_std', 'isi_viol', 'amplitude_cutoff', 'amplitude_std', 'epoch_name', 'ks2_contamination_pct', 'ks2_label' ] units_metadata_dict = self._initialize_container_dict( 'Units', default_value=list()) current_units_objects = self._get_current_object_names(units_objects) for object_name in current_units_objects: if 'clusters' in object_name: units_metadata_dict = \ self._get_dynamictable_object(self.dataset_details.copy(), 'clusters', 'Units', default_colnames_dict=dict(location='brainAcronyms', waveform_mean='waveforms', electrodes='channels', electrode_group='probes', ), drop_attrs=['uuids', 'metrics']) units_metadata_dict['Units'].extend( self._get_dynamictable_array( name=['obs_intervals', 'spike_times'], data=[ 'trials.intervals', 'spikes.clusters,spikes.times' ], description=[ 'time intervals of each cluster', 'spike times of cluster' ])) units_metadata_dict['Units'].extend( self._get_dynamictable_array( name=metrics_columns, data=['clusters.metrics'] * len(metrics_columns), description=['metrics_table columns data'] * len(metrics_columns))) return units_metadata_dict @property def electrodegroup_metadata(self): electrodes_group_metadata_dict = self._initialize_container_dict( 'ElectrodeGroup', default_value=[]) for ii in range(len(self.probe_metadata['Probes'])): try: location_str = self.probe_metadata['Probes'][ii][ 'trajectory_estimate'][0]['coordinate_system'] except: location_str = 'None' electrodes_group_metadata_dict['ElectrodeGroup'].extend( self._get_dynamictable_array( name=[self.probe_metadata['Probes'][ii]['name']], description=[ 'model {}'.format( self.probe_metadata['Probes'][ii]['model']) ], device=[self.device_metadata['Device'][0]['name']], location=[ 'Mouse CoordinateSystem:{}'.format(location_str) ])) return electrodes_group_metadata_dict @property def electrodetable_metadata(self): electrodes_objects = ['channels'] electrodes_table_metadata_dict = self._initialize_container_dict( 'ElectrodeTable') current_electrodes_objects = self._get_current_object_names( electrodes_objects) for i in current_electrodes_objects: electrodes_table_metadata_dict = self._get_dynamictable_object( self.dataset_details.copy(), 'channels', 'ElectrodeTable', default_colnames_dict=dict(group='probes', x='localCoordinates', y='localCoordinates')) return electrodes_table_metadata_dict @property def ecephys_metadata(self): ecephys_objects = [ 'templates', '_iblqc_ephysTimeRms', '_iblqc_ephysSpectralDensity' ] container_object_names = [ 'SpikeEventSeries', 'ElectricalSeries', 'Spectrum' ] custom_attrs_objects = [['waveforms'], ['rms'], ['power']] ecephys_container = self._initialize_container_dict('Ecephys') kwargs = dict() for i, j, k in zip(ecephys_objects, container_object_names, custom_attrs_objects): current_ecephys_objects = self._get_current_object_names([i]) if current_ecephys_objects: if j == 'Spectrum': kwargs = dict( name=i, power='_iblqc_ephysSpectralDensity.power', frequencies='_iblqc_ephysSpectralDensity.freqs', timestamps=None) ecephys_container['Ecephys'].update( self._get_timeseries_object(self.dataset_details.copy(), i, j, custom_attrs=k, **kwargs)) else: warnings.warn(f'could not find {i} data in eid {self.eid}') return ecephys_container @property def acquisition_metadata(self): acquisition_objects = [ 'ephysData', '_iblrig_Camera', '_iblmic_audioSpectrogram' ] container_name_objects = [ 'ElectricalSeries', 'ImageSeries', 'DecompositionSeries' ] custom_attrs_objects = [['raw.nidq', 'raw.ap', 'raw.lf'], ['raw'], ['power']] acquisition_container = self._initialize_container_dict('Acquisition') current_acquisition_objects = self._get_current_object_names( acquisition_objects) idx = [ no for no, i in enumerate(acquisition_objects) if i in current_acquisition_objects ] current_container_name_objects = [ container_name_objects[i] for i in idx ] current_custom_attrs_objects = [custom_attrs_objects[i] for i in idx] kwargs = dict() for i, j, k in zip(current_acquisition_objects, current_container_name_objects, current_custom_attrs_objects): if j == 'DecompositionSeries': kwargs = dict(name=i, metric='power', bands='_iblmic_audioSpectrogram.frequencies') acquisition_container['Acquisition'].update( self._get_timeseries_object(self.dataset_details.copy(), i, j, custom_attrs=k, **kwargs)) return acquisition_container @property def ophys_metadata(self): raise NotImplementedError @property def icephys_metadata(self): raise NotImplementedError @property def scratch_metadata(self): # this can be used to add further details about subject, lab, raise NotImplementedError @property def complete_metadata(self): metafile_dict = { **self.eid_metadata, **self.probe_metadata, **self.nwbfile_metadata, **self.sessions_metadata, **self.subject_metadata, **self.behavior_metadata, **self.trials_metadata, **self.stimulus_metadata, **self.units_metadata, **self.electrodetable_metadata, 'Ecephys': { **self.ecephys_metadata, **self.device_metadata, **self.electrodegroup_metadata, }, 'Ophys': dict(), 'Icephys': dict(), **self.acquisition_metadata } return metafile_dict def write_metadata(self, fileloc, savetype=None): if savetype is not None: if Path(fileloc).suffix != savetype: raise ValueError(f'{fileloc} should of of type {savetype}') else: savetype = Path(fileloc).suffix full_metadata = self.complete_metadata if savetype == '.json': full_metadata['NWBFile']['session_start_time'] = datetime.strftime( full_metadata['NWBFile']['session_start_time'], '%Y-%m-%dT%X') full_metadata['IBLSubject']['date_of_birth'] = datetime.strftime( full_metadata['IBLSubject']['date_of_birth'], '%Y-%m-%dT%X') with open(fileloc, 'w') as f: json.dump(full_metadata, f, indent=2) elif savetype in ['.yaml', '.yml']: with open(fileloc, 'w') as f: yaml.dump(full_metadata, f, default_flow_style=False) print(f'data written in {fileloc}')
nyq = 0.5 * fs normal_cutoff = cutoff / nyq b, a = signal.butter(order, normal_cutoff, btype=ftype, analog=False) y = signal.filtfilt(b, a, data) return y FIG_PATH = '/home/guido/Figures/DLC/' # Query sessions with available DLC data using ONE one = ONE() dtypes = [ 'camera.dlc', 'camera.times', 'trials.feedback_times', 'trials.feedbackType', 'trials.stimOn_times', 'trials.choice' ] eids = one.search(dataset_types=dtypes) # Initialize dataframes pupil_stim_on = pd.DataFrame(columns=['eid', 'timepoint', 'trace']) pupil_reward = pd.DataFrame(columns=['eid', 'timepoint', 'trace']) pupil_no_reward = pd.DataFrame(columns=['eid', 'timepoint', 'trace']) paw_left = pd.DataFrame(columns=['eid', 'timepoint', 'trace']) paw_right = pd.DataFrame(columns=['eid', 'timepoint', 'trace']) tongue_reward = pd.DataFrame(columns=['eid', 'timepoint', 'trace']) tongue_no_reward = pd.DataFrame(columns=['eid', 'timepoint', 'trace']) # Loop over sessions for i, eid in enumerate(eids): if np.mod(i + 1, 5) == 0: print('Processing DLC data of session %d of %d' % (i + 1, len(eids)))
from warnings import warn import numpy as np import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import seaborn as sns import scipy.stats as stats import pandas as pd from oneibl.one import ONE import alf.io as aio import brainbox as bb from ibllib.io.spikeglx import glob_ephys_files # Set eid and probe name # # ------------------------# one = ONE() eid = one.search(subject='ZM_2104', date='2019-09-19', number=1)[0] probe = 'probe_right' # *Note: new probe naming convention is 'probe00', 'probe01', etc. # Get important directories from `eid` # # --------------------------------------# session_path = one.path_from_eid(eid) alf_dir = Path.joinpath(session_path, 'alf') alf_probe_dir = Path.joinpath(alf_dir, probe) ephys_files = glob_ephys_files(session_path) ephys_file_path = None if ephys_files: ephys_file_path = ephys_files[0]['ap'] else: warn( 'Ephys file not found! Some of the examples in this script require ephys file,' )
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA from functions_5HT import paths, plot_settings, one_session_path from oneibl.one import ONE one = ONE() # Settings PRE_TIME = 1 POST_TIME = -0.5 MIN_TRIALS = 200 MIN_NEURONS = 20 DATA_PATH, FIG_PATH, SAVE_PATH = paths() FIG_PATH = join(FIG_PATH, 'WholeBrain') # Get list of recordings eids, ses_info = one.search(user='******', dataset_types='spikes.times', details=True) lda_result = pd.DataFrame() for i, eid in enumerate(eids): # Load in data print('Processing session %d of %d' % (i + 1, len(eids))) session_path = one_session_path(eid) trials = one.load_object(eid, 'trials') probes = one.load_object(eid, 'probes', download_only=False) if (not hasattr(trials, 'stimOn_times') or (trials.stimOn_times.shape[0] != trials.probabilityLeft.shape[0]) or (not hasattr(probes, 'trajectory'))): print('Invalid data, skipping recording') continue
# Settings MIN_CONTRAST = 0.1 ALPHA = 0.05 def one_session_path(eid): ses = one.alyx.rest('sessions', 'read', id=eid) return Path(one._par.CACHE_DIR, ses['lab'], 'Subjects', ses['subject'], ses['start_time'][:10], str(ses['number']).zfill(3)) # Get list of recordings eids, ses_info = one.search(dataset_types='spikes.times', task_protocol='_iblrig_tasks_ephysChoiceWorld', details=True) # Set path to save plots DATA_PATH, FIG_PATH, SAVE_PATH = paths() FIG_PATH = join(FIG_PATH, 'WholeBrain') resp = pd.DataFrame() for i, eid in enumerate(eids): # Load in data print('Processing session %d of %d' % (i + 1, len(eids))) session_path = one_session_path(eid) trials = one.load_object(eid, 'trials') probes = one.load_object(eid, 'probes', download_only=False) if ((not hasattr(trials, 'stimOn_times'))
""" Quick search through the Alyx database to see all mice/sessions ever used in training. """ # Author: Gaelle Chapuis import numpy from oneibl.one import ONE one = ONE() eIDs, ses = one.search(task_protocol='world', project='ibl_neuropixel_brainwide_01', details=True) sub = [p['subject'] for p in ses] su, ind_su = numpy.unique(sub, return_index=True) print(f'N subjects: {len(su)} - N sessions: {len(eIDs)}')
the terminal. In this case no plot is generated. """ from oneibl.one import ONE import numpy as np import matplotlib.pyplot as plt from ibllib.atlas import BrainRegions one = ONE() # Specify subject, date and probe we are interested in subject = 'CSHL049' date = '2020-01-08' sess_no = 1 probe_label = 'probe01' eid = one.search(subject=subject, date=date, number=sess_no)[0] _ = one.load(eid, dataset_types=['clusters.channels'], download_only=True) alf_path = one.path_from_eid(eid).joinpath('alf', probe_label) cluster_chans = np.load(alf_path.joinpath('clusters.channels.npy')) aligned_traj = one.alyx.rest('trajectories', 'list', subject=subject, session=eid, probe=probe_label, provenance='Ephys aligned histology track') if len(aligned_traj) > 0: print('Getting channels for provenance ' + aligned_traj[0]['provenance'])
""" Get spikes, clusters and channels data ======================================== Downloads and loads in spikes, clusters and channels data for a given session. Data is returned """ from oneibl.one import ONE import brainbox.io.one as bbone one = ONE() # Find eid of interest eid = one.search(subject='CSH_ZAD_001', date='2020-01-14')[0] ################################################################################################## # Example 1: # Download spikes, clusters and channels data for all available probes for this session. # The data for each probe is returned as a dict spikes, clusters, channels = bbone.load_spike_sorting_with_channel(eid, one=one) print(spikes.keys()) print(spikes['probe00'].keys()) ################################################################################################## # Example 2: # Download spikes, clusters and channels data for a single probe spikes, clusters, channels = bbone.load_spike_sorting_with_channel( eid, one=one, probe='probe00') print(spikes.keys()) ##################################################################################################
if __name__ == '__main__': from pathlib import Path from oneibl.one import ONE import alf.io as ioalf # user options BINSIZE = 0.05 # sec LAGS = 4 # number of bins for calculating RF METHOD = 'corr' # 'corr' | 'sta' # get the data from flatiron and the current folder one = ONE() eid = one.search(subject='ZM_2104', date='2019-09-19', number=1) D = one.load(eid[0], clobber=False, download_only=True) session_path = Path(D.local_path[0]).parent # load objects spikes = ioalf.load_object(session_path, 'spikes') rfmap = ioalf.load_object(session_path, '_iblcertif_.rfmap') rf_stim_times = rfmap['rfmap.times.00'] rf_stim = rfmap['rfmap.stims.00'].astype('float') # compute receptive fields if METHOD == 'sta': # method in Durand et al 2016; ~9 min for 700 units on a single cpu core print('computing receptive fields...', end='') rfs = compute_rfs( spikes.times, spikes.clusters, rf_stim_times, rf_stim, lags=LAGS, binsize=BINSIZE)
from brainbox.io.one import load_spike_sorting, load_channel_locations from oneibl.one import ONE one = ONE(base_url="https://dev.alyx.internationalbrainlab.org") eids = one.search(subject='ZM_2407', task_protocol='ephys') channels = load_channel_locations(eids[0], one=one) spikes, clusters = load_spike_sorting(eids[0], one=one)
## Load #1 dataset_types = [ 'clusters.templateWaveforms', 'clusters.probes', 'clusters.depths' ] eid = '86e27228-8708-48d8-96ed-9aa61ab951db' wf, pr, d = one.load(eid, dataset_types=dataset_types) ## Load #2 my_data = one.load(eid, dataset_types=dataset_types, dclass_output=True) from ibllib.misc import pprint pprint(my_data.local_path) pprint(my_data.dataset_type) ## Load everything eid, ses_info = one.search(subject='flowers') my_data = one.load(eid[0]) pprint(my_data.dataset_type) ## Load eid = '86e27228-8708-48d8-96ed-9aa61ab951db' dataset_types = [ 'clusters.probes', 'thisDataset.IveJustMadeUp', 'clusters.depths' ] t, empty, cl = one.load(eid, dataset_types=dataset_types) ## List #1 one.ls_dataset_types() one.ls_users() one.ls_subjects()
import matplotlib.pyplot as plt import pandas as pd from oneibl.one import ONE from ibllib.time import isostr2date # import sys # sys.path.extend('/home/owinter/PycharmProjects/WGs/BehaviourAnaysis/python') from load_mouse_data import get_behavior from behavior_plots import plot_psychometric one = ONE() # https://alyx.internationalbrainlab.org/admin/actions/session/e752b02d-b54d-4373-b51e-0b31be5f8ee5/change/ # first get the subject information subject_details = one.alyx.rest('subjects', 'read', 'IBL_14') # plot the weight curve # https://alyx.internationalbrainlab.org/admin-actions/water-history/37c8f897-cbcc-4743-bad6-764ccbbfb190 wei = pd.DataFrame(subject_details['weighings']) wei['date_time'].apply(isostr2date) wei.sort_values('date_time', inplace=True) plt.plot(wei.date_time, wei.weight) # now let's get some session information ses_ids = one.search(subjects='IBL_14', date_range='2018-11-27') print(one.list(ses_ids[0])) df = get_behavior('IBL_14', date_range='2018-11-27') plt.figure() plot_psychometric(df, ax=plt.axes(), color="orange")
''' Download raw ephys datasets for all probes in a single session via ONE. (example written for the LFP, but the download can be done for AP files similarly by replacing 'lf' with 'ap') ''' # Author: Olivier, Gaelle from ibllib.io import spikeglx from oneibl.one import ONE # === Option 1 === Download a dataset of interest one = ONE() # Get a specific session eID eid = one.search(subject='ZM_2240', date_range='2020-01-22')[0] # Define and load dataset types of interest dtypes = ['ephysData.raw.lf', # lf : LFP 'ephysData.raw.meta', 'ephysData.raw.ch', 'ephysData.raw.sync'] # Used for synchronisation one.load(eid, dataset_types=dtypes, download_only=True) # Get the files information session_path = one.path_from_eid(eid) efiles = [ef for ef in spikeglx.glob_ephys_files(session_path, bin_exists=False) if ef.get('lf', None)] efile = efiles[0]['lf'] # Example: access to the first file
from v1_protocol import plot as v1_plot import alf.io as aio import brainbox as bb # The examples below can be run independently. # Example 1: For 'ZM_2104/2019-09-19/001' 'probe_right', generate all 4 figures (grating response # summary, grating response selected, unit metrics summary, and unit metrics selected) using # default parameters. For the summary figures, use all units, and for the selected figures, use # 4 randomly chosen units. # ------------------------------------------------------------------------------------------------- # Set the eid as `eid` and probe name as `probe` - these two input args are required for running # `gen_figures` one = ONE() eid = one.search(subject='ZM_2104', date='2019-09-19', number=1)[0] probe = 'probe_right' # Get paths to the required dataset_types. If required dataset_types are not already downloaded, # download them. dtypes = [ 'clusters.amps', 'clusters.channels', 'clusters.depths', 'clusters.metrics', 'clusters.peakToTrough', 'clusters.uuids', 'clusters.waveforms', 'clusters.waveformsChannels', 'spikes.amps', 'spikes.clusters', 'spikes.depths', 'spikes.samples', 'spikes.templates', 'spikes.times', 'ephysData.raw.meta', '_spikeglx_sync.channels', '_spikeglx_sync.polarities', '_spikeglx_sync.times', '_iblrig_RFMapStim.raw', '_iblrig_taskSettings.raw', '_iblrig_codeFiles.raw' ] d_paths = one.load(eid,
class RegistrationClient: """ Object that keeps the ONE instance and provides method to create sessions and register data. """ def __init__(self, one=None): self.one = one if not one: self.one = ONE() self.dtypes = self.one.alyx.rest('dataset-types', 'list') self.registration_patterns = [ dt['filename_pattern'] for dt in self.dtypes if dt['filename_pattern'] ] self.file_extensions = [ df['file_extension'] for df in self.one.alyx.rest('data-formats', 'list') ] def create_sessions(self, root_data_folder, glob_pattern='**/create_me.flag', dry=False): """ Create sessions looking recursively for flag files :param root_data_folder: folder to look for create_me.flag :param dry: bool. Dry run if True :param glob_pattern: bool. Dry run if True :return: None """ flag_files = Path(root_data_folder).glob(glob_pattern) for flag_file in flag_files: if dry: print(flag_file) continue _logger.info('creating session for ' + str(flag_file.parent)) # providing a false flag stops the registration after session creation self.create_session(flag_file.parent) flag_file.unlink() return [ff.parent for ff in flag_files] def create_session(self, session_path): """ create_session(session_path) """ return self.register_session(session_path, file_list=False) def register_sync(self, root_data_folder, dry=False): """ Register sessions looking recursively for flag files :param root_data_folder: folder to look for register_me.flag :param dry: bool. Dry run if True :return: """ flag_files = Path(root_data_folder).glob('**/register_me.flag') for flag_file in flag_files: if dry: continue file_list = flags.read_flag_file(flag_file) _logger.info('registering ' + str(flag_file.parent)) self.register_session(flag_file.parent, file_list=file_list) flags.write_flag_file(flag_file.parent.joinpath('flatiron.flag'), file_list=file_list) flag_file.unlink() if flag_file.parent.joinpath('create_me.flag').exists(): flag_file.parent.joinpath('create_me.flag').unlink() _logger.info('registered' + '\n') def register_session(self, ses_path, file_list=True): """ Register session in Alyx :param ses_path: path to the session :param file_list: bool. Set to False will only create the session and skip registration :param repository_name: Optional, repository on which to register the data :return: Status string on error """ if isinstance(ses_path, str): ses_path = Path(ses_path) # read meta data from the rig for the session from the task settings file settings_json_file = list( ses_path.glob('**/_mflab_taskSettings.raw*.json')) if not settings_json_file: settings_json_file = list( ses_path.glob('**/_mflab_taskSettings.raw*.json')) if not settings_json_file: _logger.error( ['could not find _mflab_taskSettings.raw.json. Abort.']) return _logger.warning( [f'Settings found in a strange place: {settings_json_file}']) else: settings_json_file = settings_json_file[0] md = _read_settings_json_compatibility_enforced(settings_json_file) # query alyx endpoints for subject, error if not found try: subject = self.one.alyx.rest( 'subjects?nickname=' + md['SUBJECT_NAME'], 'list')[0] except IndexError: _logger.error( f"Subject: {md['SUBJECT_NAME']} doesn't exist in Alyx. ABORT.") raise ibllib.exceptions.AlyxSubjectNotFound(md['SUBJECT_NAME']) # look for a session from the same subject, same number on the same day session_id, session = self.one.search(subjects=subject['nickname'], date_range=md['SESSION_DATE'], number=md['SESSION_NUMBER'], details=True) print('session if exists', session_id, session) try: user = self.one.alyx.rest('users', 'read', id=md["PROTOCOL_CREATOR"][0]) except Exception as e: _logger.error( f"User: {md['PROTOCOL_CREATOR'][0]} doesn't exist in Alyx. ABORT" ) raise e username = user['username'] if user else subject['responsible_user'] # load the trials data to get information about session duration and performance ses_data = raw.load_data(ses_path) start_time, end_time = _get_session_times(ses_path, md, ses_data) n_trials, n_correct_trials = _get_session_performance(md, ses_data) # this is the generic relative path: subject/yyyy-mm-dd/NNN gen_rel_path = Path(subject['nickname'], md['SESSION_DATE'], '{0:03d}'.format(int(md['SESSION_NUMBER']))) # if nothing found create a new session in Alyx task_protocol = md['PROTOCOL'] + md['VERSION_TAG'] dset_types = md['DATASET_TYPES'] alyx_procedure = _alyx_procedure_from_task(task_protocol) if not session: ses_ = { 'subject': subject['nickname'], 'users': [username], 'procedures': [] if alyx_procedure is None else [alyx_procedure], 'lab': subject['lab'], 'project': md['PROJECT'], 'type': 'Experiment', 'task_protocol': task_protocol, 'number': md['SESSION_NUMBER'], 'start_time': ibllib.time.date2isostr(start_time), 'end_time': ibllib.time.date2isostr(end_time) if end_time else None, 'n_correct_trials': n_correct_trials, 'n_trials': n_trials, 'data_dataset_session_related': md['DATASET_TYPES'], 'dset_types': md['DATASET_TYPES'], 'json': md, } session = self.one.alyx.rest('sessions', 'create', data=ses_) #if md['SUBJECT_WEIGHT']: if 'SUBJECT_WEIGHT' in md.keys(): wei_ = { 'subject': subject['nickname'], 'date_time': ibllib.time.date2isostr(start_time), 'weight': md['SUBJECT_WEIGHT'], 'user': username } self.one.alyx.rest('weighings', 'create', data=wei_) else: # TODO: if session exists and no json partial_upgrade it print('session exists, lets update it') ses_ = { 'subject': subject['nickname'], 'users': [username], 'procedures': [] if alyx_procedure is None else [alyx_procedure], 'lab': subject['lab'], 'project': md['PROJECT'], 'type': 'Experiment', 'task_protocol': task_protocol, 'number': md['SESSION_NUMBER'], 'start_time': ibllib.time.date2isostr(start_time), 'end_time': ibllib.time.date2isostr(end_time) if end_time else None, 'n_correct_trials': n_correct_trials, 'n_trials': n_trials, 'data_dataset_session_related': md['DATASET_TYPES'], 'dset_types': md['DATASET_TYPES'], 'json': md, } print('ses', ses_) #session = self.one.alyx.rest('sessions', 'read', id=session_id[0]) print('session_id', session_id[0]) session = self.one.alyx.rest('sessions', 'partial_update', id=session_id[0], data=ses_) #can try update as well _logger.info(session['url'] + ' ') # create associated water administration if not found if not session['wateradmin_session_related'] and ses_data: wa_ = { 'subject': subject['nickname'], 'date_time': ibllib.time.date2isostr(end_time), 'water_administered': ses_data[-1]['water_delivered'] / 1000, 'water_type': md.get('REWARD_TYPE') or 'Water', 'user': username, 'session': session['url'][-36:], 'adlib': False } self.one.alyx.rest('water-administrations', 'create', data=wa_) # at this point the session has been created. If create only, exit if not file_list: return # register all files that match the Alyx patterns, warn user when files are encountered rename_files_compatibility(ses_path, md['VERSION_TAG']) F = [ ] # empty list whose keys will be relative paths and content filenames md5s = [] file_sizes = [] for fn in _glob_session(ses_path): if fn.suffix in EXCLUDED_EXTENSIONS: _logger.debug('Excluded: ', str(fn)) continue if not _check_filename_for_registration( fn, self.registration_patterns): _logger.warning('No matching dataset type for: ' + str(fn)) continue if fn.suffix not in self.file_extensions: _logger.warning( 'No matching dataformat (ie. file extension) for: ' + str(fn)) continue if not _register_bool(fn.name, file_list): _logger.debug('Not in filelist: ' + str(fn)) continue try: assert (str(gen_rel_path) in str(fn)) except AssertionError as e: strerr = 'ALF folder mismatch: data is in wrong subject/date/number folder. \n' strerr += ' Expected ' + str( gen_rel_path) + ' actual was ' + str(fn) _logger.error(strerr) raise e # extract the relative path of the file rel_path = Path(str(fn)[str(fn).find(str(gen_rel_path)):]) F.append(str(rel_path.relative_to(gen_rel_path))) file_sizes.append(fn.stat().st_size) md5s.append( hashfile.md5(fn) if fn.stat().st_size < 1024**3 else None) _logger.info('Registering ' + str(fn)) r_ = { 'created_by': username, 'path': str(gen_rel_path), 'filenames': F, 'hashes': md5s, 'filesizes': file_sizes, 'versions': [version.ibllib() for _ in F] } self.one.alyx.post('/register-file', data=r_)
import alf.io as ioalf BIN_SIZE = 0.025 # seconds SMOOTH_SIZE = 0.025 # seconds; standard deviation of gaussian kernel PCA_DIMS = 20 CCA_DIMS = PCA_DIMS N_SPLITS = 5 RNG_SEED = 0 # get the data from flatiron subject = 'KS005' date = '2019-08-30' number = 1 one = ONE() eid = one.search(subject=subject, date=date, number=number) D = one.load(eid[0], download_only=True) session_path = Path(D.local_path[0]).parent spikes = ioalf.load_object(session_path, 'spikes') clusters = ioalf.load_object(session_path, 'clusters') # channels = ioalf.load_object(session_path, 'channels') trials = ioalf.load_object(session_path, '_ibl_trials') # bin spikes and get trial IDs associated with them binned_spikes, binned_trialIDs, _ = bin_spikes_trials(spikes, trials, bin_size=0.01) # define areas brain_areas = np.unique(clusters.brainAcronyms)