def get_session_user(): ''' Determine desired 'session user' for a session. - 1st, try dj.config['custom']['session.user'] - 2nd, try dj.config['database.user'] - else, use 'unknown' TODO: multi-user / bulk ingest support ''' session_user = dj.config.get('custom', {}).get('session.user', None) session_user = (dj.config.get('database.user') if not session_user else session_user) if len(lab.Person() & {'username': session_user}): return session_user else: return 'unknown'
def mockdata(): print('populating with mock data') reload(ccf) reload(lab) reload(experiment) reload(ephys) reload(publication) try: # TODO: these should be loaded in a more 'official' way lab.Person().insert1({ 'username': '******', 'fullname': 'Dave Liu' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'VGAT-Chr2-EYFP Jax', 'gene_modification_description': 'VGAT' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'PV-ires-Cre X Ai32', 'gene_modification_description': 'PV' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'Rosa26 Cag lsl reachR-citrine 1A4 X PV-ires-Cre', 'gene_modification_description': 'reachR PV' }, skip_duplicates=True) # Subject 399752 / dl7 lab.Subject().insert1( { 'subject_id': 399752, 'username': '******', 'cage_number': 145375, 'date_of_birth': '2017-08-03', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 399752, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 399752, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-03', 'end_time': '2017-11-03', 'surgery_description': 'Headbar anterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 399752, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -4, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 399752, 'water_restriction_number': 'dl7', 'cage_number': 148861, 'wr_start_date': '2017-11-07', 'wr_start_weight': 25 }, skip_duplicates=True) # Subject 397853 / dl14 lab.Subject().insert1( { 'subject_id': 397853, 'username': '******', 'cage_number': 144545, 'date_of_birth': '2017-07-15', 'sex': 'M', 'animal_source': 'Allen Institute' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 397853, 'gene_modification': 'PV-ires-Cre X Ai32' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 397853, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-20', 'end_time': '2017-11-20', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 397853, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 397853, 'water_restriction_number': 'dl14', 'cage_number': 149595, 'wr_start_date': '2017-11-27', 'wr_start_weight': 24.1 }, skip_duplicates=True) # Subject 400480 / dl15 lab.Subject().insert1( { 'subject_id': 400480, 'username': '******', 'cage_number': 145700, 'date_of_birth': '2017-08-09', 'sex': 'M', 'animal_source': 'Allen Institute' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 400480, 'gene_modification': 'PV-ires-Cre X Ai32' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 400480, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-21', 'end_time': '2017-11-21', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 400480, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 400480, 'water_restriction_number': 'dl15', 'cage_number': 149598, 'wr_start_date': '2017-11-27', 'wr_start_weight': 27.6 }, skip_duplicates=True) # Subject 406680 / dl20 lab.Subject().insert1( { 'subject_id': 406680, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-06', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 406680, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 406680, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-04', 'end_time': '2018-01-04', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 406680, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 406680, 'water_restriction_number': 'dl20', 'cage_number': 151282, 'wr_start_date': '2018-01-10', 'wr_start_weight': 22.7 }, skip_duplicates=True) # Subject 408022 / dl21 lab.Subject().insert1( { 'subject_id': 408022, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-19', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 408022, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 408022, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-05', 'end_time': '2018-01-05', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 408022, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 408022, 'water_restriction_number': 'dl21', 'cage_number': 151283, 'wr_start_date': '2018-01-10', 'wr_start_weight': 21.1 }, skip_duplicates=True) # Subject 408021 / dl22 lab.Subject().insert1( { 'subject_id': 408021, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-19', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 408021, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 408021, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-15', 'end_time': '2018-01-15', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 408021, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 408021, 'water_restriction_number': 'dl22', 'cage_number': 151704, 'wr_start_date': '2018-01-19', 'wr_start_weight': 21 }, skip_duplicates=True) # Subject 407512 / dl24 lab.Subject().insert1( { 'subject_id': 407512, 'username': '******', 'cage_number': 151629, 'date_of_birth': '2017-10-13', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407512, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407512, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-16', 'end_time': '2018-01-16', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407512, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407512, 'water_restriction_number': 'dl24', 'cage_number': 151793, 'wr_start_date': '2018-01-22', 'wr_start_weight': 26 }, skip_duplicates=True) # 407513 / dl25 lab.Subject().insert1( { 'subject_id': 407513, 'username': '******', 'cage_number': 148636, 'date_of_birth': '2017-10-13', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407513, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407513, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-17', 'end_time': '2018-01-17', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407513, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407513, 'water_restriction_number': 'dl25', 'cage_number': 151794, 'wr_start_date': '2018-01-22', 'wr_start_weight': 25.5 }, skip_duplicates=True) # Subject 407986 / dl28 lab.Subject().insert1( { 'subject_id': 407986, 'username': '******', 'cage_number': 152268, 'date_of_birth': '2017-10-18', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407986, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407986, 'surgery_id': 1, 'username': '******', 'start_time': '2018-02-01', 'end_time': '2018-02-01', 'surgery_description': 'Headbar anterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407986, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -4, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407986, 'water_restriction_number': 'dl28', 'cage_number': 152312, 'wr_start_date': '2018-02-05', 'wr_start_weight': 19.8 }, skip_duplicates=True) # Subject 123457 / tw5 lab.Subject().insert1( { 'subject_id': 123457, 'username': '******', 'cage_number': 145375, 'date_of_birth': '2017-08-03', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 123457, 'water_restriction_number': 'tw5', 'cage_number': 148861, 'wr_start_date': '2017-11-07', 'wr_start_weight': 20.5 }, skip_duplicates=True) # Subject 412330 / tw34 lab.Subject().insert1( { 'subject_id': 412330, 'username': '******', 'cage_number': 154522, 'date_of_birth': '2017-12-05', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 412330, 'water_restriction_number': 'tw34', 'cage_number': 154522, 'wr_start_date': '2018-03-18', 'wr_start_weight': 21.0 }, skip_duplicates=True) # subject 432998 / dl55 lab.Subject().insert1( { 'subject_id': 432998, 'username': '******', 'cage_number': 160920, 'date_of_birth': '2018-07-02', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 432998, 'water_restriction_number': 'dl55', 'cage_number': 160920, 'wr_start_date': '2018-09-05', 'wr_start_weight': 21.0 }, skip_duplicates=True) # Subject 435884 / dl59 lab.Subject().insert1( { 'subject_id': 435884, 'username': '******', 'cage_number': 161908, 'date_of_birth': '2018-08-06', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 435884, 'water_restriction_number': 'dl59', 'cage_number': 154522, 'wr_start_date': '2018-09-30', 'wr_start_weight': 21.0 }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig1', 'room': '2w.334', 'rig_description': 'Training rig 1' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig2', 'room': '2w.334', 'rig_description': 'Training rig 2' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig3', 'room': '2w.334', 'rig_description': 'Training rig 3' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'RRig', 'room': '2w.334', 'rig_description': 'Recording rig' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'Ephys1', 'room': '2w.334', 'rig_description': 'Recording computer' }, skip_duplicates=True) # ---- Brain Location ---- experiment.BrainLocation.insert1( { 'brain_location_name': 'left_alm', 'brain_area': 'ALM', 'hemisphere': 'left', 'skull_reference': 'Bregma' }, skip_duplicates=True) experiment.BrainLocation.insert1( { 'brain_location_name': 'right_alm', 'brain_area': 'ALM', 'hemisphere': 'right', 'skull_reference': 'Bregma' }, skip_duplicates=True) experiment.BrainLocation.insert1( { 'brain_location_name': 'both_alm', 'brain_area': 'ALM', 'hemisphere': 'both', 'skull_reference': 'Bregma' }, skip_duplicates=True) experiment.BrainLocation.insert1( { 'brain_location_name': 'left_medulla', 'brain_area': 'Medulla', 'hemisphere': 'left', 'skull_reference': 'Bregma' }, skip_duplicates=True) experiment.BrainLocation.insert1( { 'brain_location_name': 'right_medulla', 'brain_area': 'Medulla', 'hemisphere': 'right', 'skull_reference': 'Bregma' }, skip_duplicates=True) experiment.BrainLocation.insert1( { 'brain_location_name': 'both_medulla', 'brain_area': 'Medulla', 'hemisphere': 'both', 'skull_reference': 'Bregma' }, skip_duplicates=True) # Probe (Neuropixel) npx_probe_model = '15131808323' # using Model No. - SN TBD? lab.Probe.insert1( { 'probe': npx_probe_model, 'probe_type': 'neuropixel' }, skip_duplicates=True, ) lab.Probe.Electrode.insert( ({ 'probe': npx_probe_model, 'electrode': x } for x in range(1, 961)), skip_duplicates=True, ) except Exception as e: print("error creating mock data: {e}".format(e=e), file=sys.stderr) raise
def populatemetadata(): #%% save metadata from google drive if necessairy lastmodify = online_notebook.fetch_lastmodify_time_animal_metadata() with open(dj.config['locations.metadata_surgery_experiment'] + 'last_modify_time.json') as timedata: lastmodify_prev = json.loads(timedata.read()) if lastmodify != lastmodify_prev: print('updating surgery and WR metadata from google drive') dj.config['locations.metadata_surgery_experiment'] df_surgery = online_notebook.fetch_animal_metadata() df_surgery.to_csv(dj.config['locations.metadata_surgery_experiment'] + 'Surgery.csv') IDs = df_surgery['ID'].tolist() for ID in IDs: df_wr = online_notebook.fetch_water_restriction_metadata(ID) if type(df_wr) == pd.DataFrame: df_wr.to_csv( dj.config['locations.metadata_surgery_experiment'] + ID + '.csv') with open( dj.config['locations.metadata_surgery_experiment'] + 'last_modify_time.json', "w") as write_file: json.dump(lastmodify, write_file) print('surgery and WR metadata updated') lastmodify = online_notebook.fetch_lastmodify_time_lab_metadata() with open(dj.config['locations.metadata_lab'] + 'last_modify_time.json') as timedata: lastmodify_prev = json.loads(timedata.read()) if lastmodify != lastmodify_prev: print('updating Lab metadata from google drive') dj.config['locations.metadata_lab'] IDs = ['Experimenter', 'Rig', 'Virus'] for ID in IDs: df_wr = online_notebook.fetch_lab_metadata(ID) if type(df_wr) == pd.DataFrame: df_wr.to_csv(dj.config['locations.metadata_lab'] + ID + '.csv') with open( dj.config['locations.metadata_lab'] + 'last_modify_time.json', "w") as write_file: json.dump(lastmodify, write_file) print('Lab metadata updated') #%% add users df_experimenters = pd.read_csv(dj.config['locations.metadata_lab'] + 'Experimenter.csv') experimenterdata = list() for experimenter in df_experimenters.iterrows(): experimenter = experimenter[1] dictnow = { 'username': experimenter['username'], 'fullname': experimenter['fullname'] } experimenterdata.append(dictnow) print('adding experimenters') for experimenternow in experimenterdata: try: lab.Person().insert1(experimenternow) except dj.errors.DuplicateError: print('duplicate. experimenter: ', experimenternow['username'], ' already exists') #%% add rigs df_rigs = pd.read_csv(dj.config['locations.metadata_lab'] + 'Rig.csv') rigdata = list() for rig in df_rigs.iterrows(): rig = rig[1] dictnow = { 'rig': rig['rig'], 'room': rig['room'], 'rig_description': rig['rig_description'] } rigdata.append(dictnow) print('adding rigs') for rignow in rigdata: try: lab.Rig().insert1(rignow) except dj.errors.DuplicateError: print('duplicate. rig: ', rignow['rig'], ' already exists') #%% add viruses df_viruses = pd.read_csv(dj.config['locations.metadata_lab'] + 'Virus.csv') virusdata = list() serotypedata = list() for virus in df_viruses.iterrows(): virus = virus[1] if type(virus['remarks']) != str: virus['remarks'] = '' dictnow = { 'virus_id': virus['virus_id'], 'virus_source': virus['virus_source'], 'serotype': virus['serotype'], 'username': virus['username'], 'virus_name': virus['virus_name'], 'titer': virus['titer'], 'order_date': virus['order_date'], 'remarks': virus['remarks'] } virusdata.append(dictnow) dictnow = {'serotype': virus['serotype']} serotypedata.append(dictnow) print('adding rigs') for virusnow, serotypenow in zip(virusdata, serotypedata): try: lab.Serotype().insert1(serotypenow) except dj.errors.DuplicateError: print('duplicate serotype: ', serotypenow['serotype'], ' already exists') try: lab.Virus().insert1(virusnow) except dj.errors.DuplicateError: print('duplicate virus: ', virusnow['virus_name'], ' already exists') #%% populate subjects, surgeries and water restrictions print('adding surgeries and stuff') df_surgery = pd.read_csv( dj.config['locations.metadata_surgery_experiment'] + 'Surgery.csv') #%% for item in df_surgery.iterrows(): if item[1]['project'] == dj.config['project'] and ( item[1]['status'] == 'training' or item[1]['status'] == 'sacrificed'): subjectdata = { 'subject_id': item[1]['animal#'], 'cage_number': item[1]['cage#'], 'date_of_birth': item[1]['DOB'], 'sex': item[1]['sex'], 'username': item[1]['experimenter'], } try: lab.Subject().insert1(subjectdata) except dj.errors.DuplicateError: print('duplicate. animal :', item[1]['animal#'], ' already exists') surgeryidx = 1 while 'surgery date (' + str(surgeryidx) + ')' in item[1].keys( ) and item[1]['surgery date (' + str(surgeryidx) + ')'] and type( item[1]['surgery date (' + str(surgeryidx) + ')']) == str: start_time = datetime.strptime( item[1]['surgery date (' + str(surgeryidx) + ')'] + ' ' + item[1]['surgery time (' + str(surgeryidx) + ')'], '%Y-%m-%d %H:%M') end_time = start_time + timedelta( minutes=int(item[1]['surgery length (min) (' + str(surgeryidx) + ')'])) surgerydata = { 'surgery_id': surgeryidx, 'subject_id': item[1]['animal#'], 'username': item[1]['experimenter'], 'start_time': start_time, 'end_time': end_time, 'surgery_description': item[1]['surgery type (' + str(surgeryidx) + ')'] + ':-: comments: ' + str(item[1]['surgery comments (' + str(surgeryidx) + ')']), } try: lab.Surgery().insert1(surgerydata) except dj.errors.DuplicateError: print('duplicate. surgery for animal ', item[1]['animal#'], ' already exists: ', start_time) #checking craniotomies #% cranioidx = 1 while 'craniotomy diameter (' + str( cranioidx) + ')' in item[1].keys() and item[1][ 'craniotomy diameter (' + str(cranioidx) + ')'] and ( type(item[1]['craniotomy surgery id (' + str(cranioidx) + ')']) == int or type(item[1]['craniotomy surgery id (' + str(cranioidx) + ')']) == float): if item[1]['craniotomy surgery id (' + str(cranioidx) + ')'] == surgeryidx: proceduredata = { 'surgery_id': surgeryidx, 'subject_id': item[1]['animal#'], 'procedure_id': cranioidx, 'skull_reference': item[1]['craniotomy reference (' + str(cranioidx) + ')'], 'ml_location': item[1]['craniotomy lateral (' + str(cranioidx) + ')'], 'ap_location': item[1]['craniotomy anterior (' + str(cranioidx) + ')'], 'surgery_procedure_description': 'craniotomy: ' + item[1]['craniotomy comments (' + str(cranioidx) + ')'], } try: lab.Surgery.Procedure().insert1(proceduredata) except dj.errors.DuplicateError: print('duplicate cranio for animal ', item[1]['animal#'], ' already exists: ', cranioidx) cranioidx += 1 #% virusinjidx = 1 while 'virus inj surgery id (' + str( virusinjidx) + ')' in item[1].keys() and item[1][ 'virus inj virus id (' + str(virusinjidx) + ')'] and item[1]['virus inj surgery id (' + str(virusinjidx) + ')']: if item[1]['virus inj surgery id (' + str(virusinjidx) + ')'] == surgeryidx: # ============================================================================= # print('waiting') # timer.sleep(1000) # ============================================================================= if '[' in item[1]['virus inj lateral (' + str(virusinjidx) + ')']: virus_ml_locations = eval( item[1]['virus inj lateral (' + str(virusinjidx) + ')']) virus_ap_locations = eval( item[1]['virus inj anterior (' + str(virusinjidx) + ')']) virus_dv_locations = eval( item[1]['virus inj ventral (' + str(virusinjidx) + ')']) virus_volumes = eval( item[1]['virus inj volume (nl) (' + str(virusinjidx) + ')']) else: virus_ml_locations = [ int(item[1]['virus inj lateral (' + str(virusinjidx) + ')']) ] virus_ap_locations = [ int(item[1]['virus inj anterior (' + str(virusinjidx) + ')']) ] virus_dv_locations = [ int(item[1]['virus inj ventral (' + str(virusinjidx) + ')']) ] virus_volumes = [ int(item[1]['virus inj volume (nl) (' + str(virusinjidx) + ')']) ] for virus_ml_location, virus_ap_location, virus_dv_location, virus_volume in zip( virus_ml_locations, virus_ap_locations, virus_dv_locations, virus_volumes): injidx = len(lab.Surgery.VirusInjection() & surgerydata) + 1 virusinjdata = { 'surgery_id': surgeryidx, 'subject_id': item[1]['animal#'], 'injection_id': injidx, 'virus_id': item[1]['virus inj virus id (' + str(virusinjidx) + ')'], 'skull_reference': item[1]['virus inj reference (' + str(virusinjidx) + ')'], 'ml_location': virus_ml_location, 'ap_location': virus_ap_location, 'dv_location': virus_dv_location, 'volume': virus_volume, 'dilution': item[1]['virus inj dilution (' + str(virusinjidx) + ')'], 'description': 'virus injection: ' + item[1]['virus inj comments (' + str(virusinjidx) + ')'], } try: lab.Surgery.VirusInjection().insert1( virusinjdata) except dj.errors.DuplicateError: print('duplicate virus injection for animal ', item[1]['animal#'], ' already exists: ', injidx) virusinjidx += 1 #% surgeryidx += 1 #% if item[1]['ID']: #df_wr = online_notebook.fetch_water_restriction_metadata(item[1]['ID']) try: df_wr = pd.read_csv( dj.config['locations.metadata_surgery_experiment'] + item[1]['ID'] + '.csv') except: df_wr = None if type(df_wr) == pd.DataFrame: wrdata = { 'subject_id': item[1]['animal#'], 'water_restriction_number': item[1]['ID'], 'cage_number': item[1]['cage#'], 'wr_start_date': df_wr['Date'][0], 'wr_start_weight': df_wr['Weight'][0], } try: lab.WaterRestriction().insert1(wrdata) except dj.errors.DuplicateError: print('duplicate. water restriction :', item[1]['animal#'], ' already exists')
def populatemetadata(): #%% save metadata from google drive if necessairy lastmodify = online_notebook.fetch_lastmodify_time_animal_metadata() #print(lastmodify) with open(dj.config['locations.metadata_behavior'] + 'last_modify_time.json') as timedata: lastmodify_prev = json.loads(timedata.read()) if lastmodify != lastmodify_prev: print('updating surgery and WR metadata from google drive') dj.config['locations.metadata_behavior'] df_surgery = online_notebook.fetch_animal_metadata() df_surgery.to_csv(dj.config['locations.metadata_behavior'] + 'Surgery.csv') IDs = df_surgery['ID'].tolist() for ID in IDs: df_wr = online_notebook.fetch_water_restriction_metadata(ID) if type(df_wr) == pd.DataFrame: df_wr.to_csv(dj.config['locations.metadata_behavior'] + ID + '.csv') with open( dj.config['locations.metadata_behavior'] + 'last_modify_time.json', "w") as write_file: json.dump(lastmodify, write_file) print('surgery and WR metadata updated') lastmodify = online_notebook.fetch_lastmodify_time_lab_metadata() with open(dj.config['locations.metadata_lab'] + 'last_modify_time.json') as timedata: lastmodify_prev = json.loads(timedata.read()) if lastmodify != lastmodify_prev: print('updating Lab metadata from google drive') dj.config['locations.metadata_lab'] IDs = ['Experimenter', 'Rig'] for ID in IDs: df_wr = online_notebook.fetch_lab_metadata(ID) if type(df_wr) == pd.DataFrame: df_wr.to_csv(dj.config['locations.metadata_lab'] + ID + '.csv') with open( dj.config['locations.metadata_lab'] + 'last_modify_time.json', "w") as write_file: json.dump(lastmodify, write_file) print('Lab metadata updated') #%% add users df_experimenters = pd.read_csv(dj.config['locations.metadata_lab'] + 'Experimenter.csv') experimenterdata = list() for experimenter in df_experimenters.iterrows(): experimenter = experimenter[1] dictnow = { 'username': experimenter['username'], 'fullname': experimenter['fullname'] } experimenterdata.append(dictnow) print('adding experimenters') for experimenternow in experimenterdata: try: lab.Person().insert1(experimenternow) except: # dj.DuplicateError: print('duplicate. experimenter: ', experimenternow['username'], ' already exists') #%% add rigs df_rigs = pd.read_csv(dj.config['locations.metadata_lab'] + 'Rig.csv') rigdata = list() for rig in df_rigs.iterrows(): rig = rig[1] dictnow = { 'rig': rig['rig'], 'room': rig['room'], 'rig_description': rig['rig_description'] } rigdata.append(dictnow) print('adding rigs') for rignow in rigdata: try: lab.Rig().insert1(rignow) except dj.errors.DuplicateError: print('duplicate. rig: ', rignow['rig'], ' already exists') #%% populate subjects, surgeries and water restrictions print('adding surgeries and stuff') df_surgery = pd.read_csv(dj.config['locations.metadata_behavior'] + 'Surgery.csv') #%% for item in df_surgery.iterrows(): if item[1]['status'] == 'experiment': subjectdata = { 'subject_id': int(item[1]['animal#']), 'username': item[1]['experimenter'], 'cage_number': item[1]['cage#'], 'date_of_birth': item[1]['DOB'], 'sex': item[1]['sex'], 'animal_source': None, } #print(subjectdata) try: lab.Subject.insert1(subjectdata) except dj.errors.DuplicateError: print('duplicate. animal :', item[1]['animal#'], ' already exists') surgeryidx = 1 #while 'surgery date ('+str(surgeryidx)+')' in item[1].keys() and item[1]['surgery date ('+str(surgeryidx)+')'] and type(item[1]['surgery date ('+str(surgeryidx)+')']) == str: # start_time = datetime.strptime(item[1]['surgery date ('+str(surgeryidx)+')']+' '+item[1]['surgery time ('+str(surgeryidx)+')'],'%Y-%m-%d %H:%M') # end_time = start_time + timedelta(minutes = int(item[1]['surgery length (min) ('+str(surgeryidx)+')'])) surgerydata = { 'surgery_id': surgeryidx, 'subject_id': item[1]['animal#'], 'username': item[1]['experimenter'], 'surgery_description': 'lesion', } # 'subject_id':int(item[1]['animal#']), # 'username': item[1]['experimenter'], # 'brain_area': item[1]['BrainArea'], # 'hemisphere': item[1]['Hemisphere'], try: lab.Surgery.insert1(surgerydata) except dj.errors.DuplicateError: print('duplicate. surgery for animal ', item[1]['animal#'], ' already exists: ') surgerylesiondata = { 'surgery_id': surgeryidx, 'lesion_id': surgeryidx, 'subject_id': item[1]['animal#'], 'method': item[1]['LesionMethod'] } try: lab.Surgery.Lesion.insert1(surgerylesiondata) except dj.errors.DuplicateError: print('duplicate. surgery lesion animal ', item[1]['animal#'], ' already exists: ') brainareadata = { #'surgery_id': surgeryidx, 'brain_area': item[1]['BrainArea'], 'subject_id': item[1]['animal#'], } try: lab.BrainArea.insert1(brainareadata) except dj.errors.DuplicateError: print('duplicate. surgery brain area animal ', item[1]['animal#'], ' already exists: ') hemispheredata = { #'surgery_id': surgeryidx, 'hemisphere': item[1]['Hemisphere'], 'subject_id': item[1]['animal#'], } try: lab.Hemisphere.insert1(hemispheredata) except dj.errors.DuplicateError: print('duplicate. surgery hemisphere area animal ', item[1]['animal#'], ' already exists: ') trainingmethoddata = { 'training_method': item[1]['TrainingMethod'], 'subject_id': item[1]['animal#'], } try: lab.Training.insert1(trainingmethoddata) except dj.errors.DuplicateError: print('duplicate. training animal ', item[1]['animal#'], ' already exists: ') #print(surgerydata) #%% #checking craniotomies #% # cranioidx = 1 # while 'craniotomy diameter ('+str(cranioidx)+')' in item[1].keys() and item[1]['craniotomy diameter ('+str(cranioidx)+')'] and (type(item[1]['craniotomy surgery id ('+str(cranioidx)+')']) == int or type(item[1]['craniotomy surgery id ('+str(cranioidx)+')']) == float): # if item[1]['craniotomy surgery id ('+str(cranioidx)+')'] == surgeryidx: # proceduredata = { # 'surgery_id': surgeryidx, # 'subject_id':item[1]['animal#'], # 'procedure_id':cranioidx, # 'skull_reference':item[1]['craniotomy reference ('+str(cranioidx)+')'], # 'ml_location':item[1]['craniotomy lateral ('+str(cranioidx)+')'], # 'ap_location':item[1]['craniotomy anterior ('+str(cranioidx)+')'], # 'surgery_procedure_description': 'craniotomy: ' + item[1]['craniotomy comments ('+str(cranioidx)+')'], # } # try: # lab.Surgery.Procedure().insert1(proceduredata) # except dj.DuplicateError: # print('duplicate cranio for animal ',item[1]['animal#'], ' already exists: ', cranioidx) # cranioidx += 1 # #% # virusinjidx = 1 # while 'virus inj surgery id ('+str(virusinjidx)+')' in item[1].keys() and item[1]['virus inj virus id ('+str(virusinjidx)+')'] and item[1]['virus inj surgery id ('+str(virusinjidx)+')']: # if item[1]['virus inj surgery id ('+str(virusinjidx)+')'] == surgeryidx: # # ============================================================================= # # print('waiting') # # timer.sleep(1000) # # ============================================================================= # if '[' in item[1]['virus inj lateral ('+str(virusinjidx)+')']: # virus_ml_locations = eval(item[1]['virus inj lateral ('+str(virusinjidx)+')']) # virus_ap_locations = eval(item[1]['virus inj anterior ('+str(virusinjidx)+')']) # virus_dv_locations = eval(item[1]['virus inj ventral ('+str(virusinjidx)+')']) # virus_volumes = eval(item[1]['virus inj volume (nl) ('+str(virusinjidx)+')']) # else: # virus_ml_locations = [int(item[1]['virus inj lateral ('+str(virusinjidx)+')'])] # virus_ap_locations = [int(item[1]['virus inj anterior ('+str(virusinjidx)+')'])] # virus_dv_locations = [int(item[1]['virus inj ventral ('+str(virusinjidx)+')'])] # virus_volumes = [int(item[1]['virus inj volume (nl) ('+str(virusinjidx)+')'])] # # for virus_ml_location,virus_ap_location,virus_dv_location,virus_volume in zip(virus_ml_locations,virus_ap_locations,virus_dv_locations,virus_volumes): # injidx = len(lab.Surgery.VirusInjection() & surgerydata) +1 # virusinjdata = { # 'surgery_id': surgeryidx, # 'subject_id':item[1]['animal#'], # 'injection_id':injidx, # 'virus_id':item[1]['virus inj virus id ('+str(virusinjidx)+')'], # 'skull_reference':item[1]['virus inj reference ('+str(virusinjidx)+')'], # 'ml_location':virus_ml_location, # 'ap_location':virus_ap_location, # 'dv_location':virus_dv_location, # 'volume':virus_volume, # 'dilution':item[1]['virus inj dilution ('+str(virusinjidx)+')'], # 'description': 'virus injection: ' + item[1]['virus inj comments ('+str(virusinjidx)+')'], # } # try: # lab.Surgery.VirusInjection().insert1(virusinjdata) # except dj.DuplicateError: # print('duplicate virus injection for animal ',item[1]['animal#'], ' already exists: ', injidx) # virusinjidx += 1 # #% # # surgeryidx += 1 #%% if item[1]['ID']: #df_wr = online_notebook.fetch_water_restriction_metadata(item[1]['ID']) try: df_wr = pd.read_csv( dj.config['locations.metadata_behavior'] + item[1]['ID'] + '.csv') except: df_wr = None if type(df_wr) == pd.DataFrame: wrdata = { 'water_restriction_number': item[1]['ID'], 'subject_id': int(item[1]['animal#']), 'cage_number': item[1]['cage#'], 'wr_start_date': '0001-01-01', 'wr_start_weight': 0, } try: lab.WaterRestriction().insert1(wrdata) except dj.errors.DuplicateError: print('duplicate. water restriction :', item[1]['animal#'], ' already exists')
def load_meta_foraging(): ''' Load metadata for the foraging task Adapted from Marton's code: https://github.com/rozmar/DataPipeline/blob/master/ingest/datapipeline_metadata.py ''' import pathlib meta_dir = dj.config.get('custom', {}).get('behavior_bpod', []).get('meta_dir') meta_lab_dir = dj.config.get('custom', {}).get('behavior_bpod', []).get('meta_lab_dir') # --- Add experimenters --- print('Adding experimenters...') df_experimenters = pd.read_csv( pathlib.Path(meta_lab_dir) / 'Experimenter.csv') duplicate_num = 0 for experimenter in df_experimenters.iterrows(): experimenter = experimenter[1] experimenternow = { 'username': experimenter['username'], 'fullname': experimenter['fullname'] } try: lab.Person().insert1(experimenternow) print(' added experimenter: ', experimenternow['username']) except dj.errors.DuplicateError: duplicate_num += 1 # print(' duplicate. experimenter: ',experimenternow['username'], ' already exists') print(f' {duplicate_num} experimenters already exist') # --- Add rigs --- print('Adding rigs... ') df_rigs = pd.read_csv(pathlib.Path(meta_lab_dir) / 'Rig.csv') duplicate_num = 0 for rig in df_rigs.iterrows(): rig = rig[1] rignow = { 'rig': rig['rig'], 'room': rig['room'], 'rig_description': rig['rig_description'] } try: lab.Rig().insert1(rignow) print(' added rig: ', rignow['rig']) except dj.errors.DuplicateError: duplicate_num += 1 # print(' duplicate. rig: ',rignow['rig'], ' already exists') print(f' {duplicate_num} rigs already exist') # --- Add viruses --- # Not implemented for now. Han # --- Add subjects and water restrictions --- print('Adding subjects and water restrictions...') df_surgery = pd.read_csv(pathlib.Path(meta_dir) / 'Surgery.csv') # For each entry duplicate_subject_num = 0 duplicate_WR_num = 0 for item in df_surgery.iterrows(): item = item[1] if item['project'] == 'foraging' and (item['status'] == 'training' or item['status'] == 'sacrificed'): # -- Add lab.Subject() -- subjectdata = { 'subject_id': item['animal#'], 'cage_number': item['cage#'], 'date_of_birth': item['DOB'], 'sex': item['sex'], 'username': item['experimenter'], } try: lab.Subject().insert1(subjectdata) print(' added subject: ', item['animal#']) except dj.errors.DuplicateError: duplicate_subject_num += 1 # print(' duplicate. animal :',item['animal#'], ' already exists') # -- Add lab.Surgery() -- # Not implemented. Han # -- Virus injection -- # Not implemented. Han # -- Add lab.WaterRestriction() -- if item['ID']: # Get water restriction start date and weight subject_csv = pathlib.Path(meta_dir) / '{}.csv'.format( item['ID']) if subject_csv.exists(): df_wr = pd.read_csv(subject_csv) else: print(' No metadata csv found for {}'.format(item['ID'])) continue wrdata = { 'subject_id': item['animal#'], 'water_restriction_number': item['ID'], 'cage_number': item['cage#'], 'wr_start_date': df_wr['Date'][0], 'wr_start_weight': df_wr['Weight'][0], } try: lab.WaterRestriction().insert1(wrdata) print(' added WR: ', item['ID']) except dj.errors.DuplicateError: duplicate_WR_num += 1 # print(' duplicate. water restriction:', item['ID'], ' already exists') print( f' {duplicate_subject_num} subjects and {duplicate_WR_num} WRs already exist' )
def mockdata(): print('populating with mock data') reload(ccf) reload(lab) reload(experiment) reload(ephys) reload(publication) try: # TODO: these should be loaded in a more 'official' way lab.Person().insert1({ 'username': '******', 'fullname': 'Dave Liu' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'VGAT-Chr2-EYFP Jax', 'gene_modification_description': 'VGAT' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'PV-ires-Cre X Ai32', 'gene_modification_description': 'PV' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'Rosa26 Cag lsl reachR-citrine 1A4 X PV-ires-Cre', 'gene_modification_description': 'reachR PV' }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 399752, 'username': '******', 'cage_number': 145375, 'date_of_birth': '2017-08-03', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 399752, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 399752, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-03', 'end_time': '2017-11-03', 'surgery_description': 'Headbar anterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 399752, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -4, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 399752, 'water_restriction_number': 'dl7', 'cage_number': 148861, 'wr_start_date': '2017-11-07', 'wr_start_weight': 25 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 397853, 'username': '******', 'cage_number': 144545, 'date_of_birth': '2017-07-15', 'sex': 'M', 'animal_source': 'Allen Institute' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 397853, 'gene_modification': 'PV-ires-Cre X Ai32' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 397853, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-20', 'end_time': '2017-11-20', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 397853, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 397853, 'water_restriction_number': 'dl14', 'cage_number': 149595, 'wr_start_date': '2017-11-27', 'wr_start_weight': 24.1 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 400480, 'username': '******', 'cage_number': 145700, 'date_of_birth': '2017-08-09', 'sex': 'M', 'animal_source': 'Allen Institute' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 400480, 'gene_modification': 'PV-ires-Cre X Ai32' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 400480, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-21', 'end_time': '2017-11-21', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 400480, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 400480, 'water_restriction_number': 'dl15', 'cage_number': 149598, 'wr_start_date': '2017-11-27', 'wr_start_weight': 27.6 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 406680, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-06', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 406680, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 406680, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-04', 'end_time': '2018-01-04', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 406680, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 406680, 'water_restriction_number': 'dl20', 'cage_number': 151282, 'wr_start_date': '2018-01-10', 'wr_start_weight': 22.7 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 408022, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-19', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 408022, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 408022, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-05', 'end_time': '2018-01-05', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 408022, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 408022, 'water_restriction_number': 'dl21', 'cage_number': 151283, 'wr_start_date': '2018-01-10', 'wr_start_weight': 21.1 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 408021, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-19', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 408021, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 408021, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-15', 'end_time': '2018-01-15', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 408021, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 408021, 'water_restriction_number': 'dl22', 'cage_number': 151704, 'wr_start_date': '2018-01-19', 'wr_start_weight': 21 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 407512, 'username': '******', 'cage_number': 151629, 'date_of_birth': '2017-10-13', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407512, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407512, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-16', 'end_time': '2018-01-16', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407512, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407512, 'water_restriction_number': 'dl24', 'cage_number': 151793, 'wr_start_date': '2018-01-22', 'wr_start_weight': 26 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 407513, 'username': '******', 'cage_number': 148636, 'date_of_birth': '2017-10-13', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407513, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407513, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-17', 'end_time': '2018-01-17', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407513, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407513, 'water_restriction_number': 'dl25', 'cage_number': 151794, 'wr_start_date': '2018-01-22', 'wr_start_weight': 25.5 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 407986, 'username': '******', 'cage_number': 152268, 'date_of_birth': '2017-10-18', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407986, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407986, 'surgery_id': 1, 'username': '******', 'start_time': '2018-02-01', 'end_time': '2018-02-01', 'surgery_description': 'Headbar anterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407986, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -4, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407986, 'water_restriction_number': 'dl28', 'cage_number': 152312, 'wr_start_date': '2018-02-05', 'wr_start_weight': 19.8 }, skip_duplicates=True) lab.Subject().insert1( { 'subject_id': 123457, 'username': '******', 'cage_number': 145375, 'date_of_birth': '2017-08-03', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 123457, 'water_restriction_number': 'tw5', 'cage_number': 148861, 'wr_start_date': '2017-11-07', 'wr_start_weight': 20.5 }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig1', 'room': '2w.334', 'rig_description': 'Training rig 1' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig2', 'room': '2w.334', 'rig_description': 'Training rig 2' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig3', 'room': '2w.334', 'rig_description': 'Training rig 3' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'RRig', 'room': '2w.334', 'rig_description': 'Recording rig' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'Ephys1', 'room': '2w.334', 'rig_description': 'Recording computer' }, skip_duplicates=True) except Exception as e: print("error creating mock data: {e}".format(e=e), file=sys.stderr) raise
def mockdata(): print('populating with mock data') reload(lab) reload(ccf) reload(experiment) reload(ephys) reload(histology) reload(tracking) try: lab.Person().insert1({ 'username': '******', 'fullname': 'Unknown' }, skip_duplicates=True) lab.Person().insert1({ 'username': '******', 'fullname': 'Dave Liu' }, skip_duplicates=True) lab.Person().insert1({ 'username': '******', 'fullname': 'Susu Chen' }, skip_duplicates=True) lab.Person().insert1({ 'username': '******', 'fullname': 'Thinh Nguyen' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'VGAT-Chr2-EYFP Jax', 'gene_modification_description': 'VGAT' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'PV-ires-Cre X Ai32', 'gene_modification_description': 'PV' }, skip_duplicates=True) lab.ModifiedGene().insert1( { 'gene_modification': 'Rosa26 Cag lsl reachR-citrine 1A4 X PV-ires-Cre', 'gene_modification_description': 'reachR PV' }, skip_duplicates=True) # Subject 399752 / dl7 lab.Subject().insert1( { 'subject_id': 399752, 'username': '******', 'cage_number': 145375, 'date_of_birth': '2017-08-03', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 399752, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 399752, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-03', 'end_time': '2017-11-03', 'surgery_description': 'Headbar anterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 399752, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -4, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 399752, 'water_restriction_number': 'dl7', 'cage_number': 148861, 'wr_start_date': '2017-11-07', 'wr_start_weight': 25 }, skip_duplicates=True) # Subject 397853 / dl14 lab.Subject().insert1( { 'subject_id': 397853, 'username': '******', 'cage_number': 144545, 'date_of_birth': '2017-07-15', 'sex': 'M', 'animal_source': 'Allen Institute' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 397853, 'gene_modification': 'PV-ires-Cre X Ai32' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 397853, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-20', 'end_time': '2017-11-20', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 397853, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 397853, 'water_restriction_number': 'dl14', 'cage_number': 149595, 'wr_start_date': '2017-11-27', 'wr_start_weight': 24.1 }, skip_duplicates=True) # Subject 400480 / dl15 lab.Subject().insert1( { 'subject_id': 400480, 'username': '******', 'cage_number': 145700, 'date_of_birth': '2017-08-09', 'sex': 'M', 'animal_source': 'Allen Institute' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 400480, 'gene_modification': 'PV-ires-Cre X Ai32' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 400480, 'surgery_id': 1, 'username': '******', 'start_time': '2017-11-21', 'end_time': '2017-11-21', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 400480, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 400480, 'water_restriction_number': 'dl15', 'cage_number': 149598, 'wr_start_date': '2017-11-27', 'wr_start_weight': 27.6 }, skip_duplicates=True) # Subject 406680 / dl20 lab.Subject().insert1( { 'subject_id': 406680, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-06', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 406680, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 406680, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-04', 'end_time': '2018-01-04', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 406680, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 406680, 'water_restriction_number': 'dl20', 'cage_number': 151282, 'wr_start_date': '2018-01-10', 'wr_start_weight': 22.7 }, skip_duplicates=True) # Subject 408022 / dl21 lab.Subject().insert1( { 'subject_id': 408022, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-19', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 408022, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 408022, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-05', 'end_time': '2018-01-05', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 408022, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 408022, 'water_restriction_number': 'dl21', 'cage_number': 151283, 'wr_start_date': '2018-01-10', 'wr_start_weight': 21.1 }, skip_duplicates=True) # Subject 408021 / dl22 lab.Subject().insert1( { 'subject_id': 408021, 'username': '******', 'cage_number': 148859, 'date_of_birth': '2017-10-19', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 408021, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 408021, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-15', 'end_time': '2018-01-15', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 408021, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 408021, 'water_restriction_number': 'dl22', 'cage_number': 151704, 'wr_start_date': '2018-01-19', 'wr_start_weight': 21 }, skip_duplicates=True) # Subject 407512 / dl24 lab.Subject().insert1( { 'subject_id': 407512, 'username': '******', 'cage_number': 151629, 'date_of_birth': '2017-10-13', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407512, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407512, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-16', 'end_time': '2018-01-16', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407512, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407512, 'water_restriction_number': 'dl24', 'cage_number': 151793, 'wr_start_date': '2018-01-22', 'wr_start_weight': 26 }, skip_duplicates=True) # 407513 / dl25 lab.Subject().insert1( { 'subject_id': 407513, 'username': '******', 'cage_number': 148636, 'date_of_birth': '2017-10-13', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407513, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407513, 'surgery_id': 1, 'username': '******', 'start_time': '2018-01-17', 'end_time': '2018-01-17', 'surgery_description': 'Headbar posterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407513, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -1.75, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407513, 'water_restriction_number': 'dl25', 'cage_number': 151794, 'wr_start_date': '2018-01-22', 'wr_start_weight': 25.5 }, skip_duplicates=True) # Subject 407986 / dl28 lab.Subject().insert1( { 'subject_id': 407986, 'username': '******', 'cage_number': 152268, 'date_of_birth': '2017-10-18', 'sex': 'F', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.Subject.GeneModification().insert1( { 'subject_id': 407986, 'gene_modification': 'VGAT-Chr2-EYFP Jax' }, skip_duplicates=True) lab.Surgery().insert1( { 'subject_id': 407986, 'surgery_id': 1, 'username': '******', 'start_time': '2018-02-01', 'end_time': '2018-02-01', 'surgery_description': 'Headbar anterior' }, skip_duplicates=True) lab.Surgery.Procedure().insert1( { 'subject_id': 407986, 'surgery_id': 1, 'procedure_id': 1, 'skull_reference': 'Bregma', 'ml_location': 0, 'ap_location': -4, 'surgery_procedure_description': 'Fiducial marker' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 407986, 'water_restriction_number': 'dl28', 'cage_number': 152312, 'wr_start_date': '2018-02-05', 'wr_start_weight': 19.8 }, skip_duplicates=True) # Subject 123457 / tw5 lab.Subject().insert1( { 'subject_id': 123457, 'username': '******', 'cage_number': 145375, 'date_of_birth': '2017-08-03', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 123457, 'water_restriction_number': 'tw5', 'cage_number': 148861, 'wr_start_date': '2017-11-07', 'wr_start_weight': 20.5 }, skip_duplicates=True) # Subject 412330 / tw34 lab.Subject().insert1( { 'subject_id': 412330, 'username': '******', 'cage_number': 154522, 'date_of_birth': '2017-12-05', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 412330, 'water_restriction_number': 'tw34', 'cage_number': 154522, 'wr_start_date': '2018-03-18', 'wr_start_weight': 21.0 }, skip_duplicates=True) # subject 432998 / dl55 lab.Subject().insert1( { 'subject_id': 432998, 'username': '******', 'cage_number': 160920, 'date_of_birth': '2018-07-02', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 432998, 'water_restriction_number': 'dl55', 'cage_number': 160920, 'wr_start_date': '2018-09-05', 'wr_start_weight': 21.0 }, skip_duplicates=True) # Subject 435884 / dl59 lab.Subject().insert1( { 'subject_id': 435884, 'username': '******', 'cage_number': 161908, 'date_of_birth': '2018-08-06', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 435884, 'water_restriction_number': 'dl59', 'cage_number': 154522, 'wr_start_date': '2018-09-30', 'wr_start_weight': 21.0 }, skip_duplicates=True) # Subject 432572 / dl56 lab.Subject().insert1( { 'subject_id': 432572, 'username': '******', 'cage_number': 161125, 'date_of_birth': '2018-06-28', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 432572, 'water_restriction_number': 'dl56', 'cage_number': 161125, 'wr_start_date': '2018-09-10', 'wr_start_weight': 21.0 }, skip_duplicates=True) # Subject 412753 / dl36 lab.Subject().insert1( { 'subject_id': 412753, 'username': '******', 'cage_number': 154570, 'date_of_birth': '2017-12-07', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 412753, 'water_restriction_number': 'dl36', 'cage_number': 154570, 'wr_start_date': '2017-03-30', 'wr_start_weight': 21.0 }, skip_duplicates=True) # Subject 440010 / dl62 lab.Subject().insert1( { 'subject_id': 440010, 'username': '******', 'cage_number': 163782, 'date_of_birth': '2018-09-24', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 440010, 'water_restriction_number': 'dl62', 'cage_number': 163782, 'wr_start_date': '2018-11-24', 'wr_start_weight': 23.0 }, skip_duplicates=True) # Subject 55004 / DL004 lab.Subject().insert1( { 'subject_id': 550004, 'username': '******', 'cage_number': 163782, 'date_of_birth': '2018-09-24', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 550004, 'water_restriction_number': 'DL004', 'cage_number': 163782, 'wr_start_date': '2018-11-24', 'wr_start_weight': 23.0 }, skip_duplicates=True) # Subject 550009 / DL009 lab.Subject().insert1( { 'subject_id': 550009, 'username': '******', 'cage_number': 163782, 'date_of_birth': '2018-09-24', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 550009, 'water_restriction_number': 'DL009', 'cage_number': 163782, 'wr_start_date': '2018-11-24', 'wr_start_weight': 23.0 }, skip_duplicates=True) # Subject 550010 / DL010 lab.Subject().insert1( { 'subject_id': 550010, 'username': '******', 'cage_number': 163782, 'date_of_birth': '2018-09-24', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 550010, 'water_restriction_number': 'DL010', 'cage_number': 163782, 'wr_start_date': '2018-11-24', 'wr_start_weight': 23.0 }, skip_duplicates=True) # Subject 550022 / DL022 lab.Subject().insert1( { 'subject_id': 550022, 'username': '******', 'cage_number': 163782, 'date_of_birth': '2018-09-24', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 550022, 'water_restriction_number': 'DL022', 'cage_number': 163782, 'wr_start_date': '2018-11-24', 'wr_start_weight': 23.0 }, skip_duplicates=True) # Subject 440959 / SC011 lab.Subject().insert1( { 'subject_id': 440959, 'username': '******', 'cage_number': 440959, 'date_of_birth': '2018-10-09', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 440959, 'water_restriction_number': 'SC011', 'cage_number': 440959, 'wr_start_date': '2018-12-21', 'wr_start_weight': 22.8 }, skip_duplicates=True) # Subject 442571 / SC022 lab.Subject().insert1( { 'subject_id': 442571, 'username': '******', 'cage_number': 442571, 'date_of_birth': '2018-10-29', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 442571, 'water_restriction_number': 'SC022', 'cage_number': 442571, 'wr_start_date': '2019-01-02', 'wr_start_weight': 26.5 }, skip_duplicates=True) # Subject 460432 / SC030 lab.Subject().insert1( { 'subject_id': 456773, 'username': '******', 'cage_number': 171857, 'date_of_birth': '2019-05-29', 'sex': 'M', 'animal_source': 'Jackson Labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 456773, 'water_restriction_number': 'SC030', 'cage_number': 171857, 'wr_start_date': '2019-08-08', 'wr_start_weight': 25.700 }, skip_duplicates=True) # Subject 460432 / SC032 lab.Subject().insert1( { 'subject_id': 460432, 'username': '******', 'cage_number': 173167, 'date_of_birth': '2019-07-15', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 460432, 'water_restriction_number': 'SC032', 'cage_number': 173167, 'wr_start_date': '2019-09-20', 'wr_start_weight': 22.8 }, skip_duplicates=True) for num in range(1, 20): lab.Subject().insert1( { 'subject_id': 777000 + num, 'username': '******', 'cage_number': 173167, 'date_of_birth': '2019-07-15', 'sex': 'M', 'animal_source': 'Jackson labs' }, skip_duplicates=True) lab.WaterRestriction().insert1( { 'subject_id': 777000 + num, 'water_restriction_number': 'FOR' + f'{num:02}', 'cage_number': 173167, 'wr_start_date': '2019-09-20', 'wr_start_weight': 22.8 }, skip_duplicates=True) # Rig lab.Rig().insert1( { 'rig': 'TRig1', 'room': '2w.334', 'rig_description': 'Training rig 1' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig2', 'room': '2w.334', 'rig_description': 'Training rig 2' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'TRig3', 'room': '2w.334', 'rig_description': 'Training rig 3' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'RRig', 'room': '2w.334', 'rig_description': 'Recording rig' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'RRig2', 'room': '2w.334', 'rig_description': 'Recording rig2' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'RRig3', 'room': '2w.334', 'rig_description': 'Recording rig3 for multi-target-licking' }, skip_duplicates=True) lab.Rig().insert1( { 'rig': 'Ephys1', 'room': '2w.334', 'rig_description': 'Recording computer' }, skip_duplicates=True) lab.ProbeType.create_neuropixels_probe() except Exception as e: print("error creating mock data: {e}".format(e=e), file=sys.stderr) raise