def make(self, key):
'''
Ephys .make() function
'''
log.info('EphysIngest().make(): key: {k}'.format(k=key))
#
# Find corresponding BehaviorIngest
#
# ... we are keying times, sessions, etc from behavior ingest;
# so lookup behavior ingest for session id, quit with warning otherwise
#
try:
behavior = (behavior_ingest.BehaviorIngest() & key).fetch1()
except dj.DataJointError:
log.warning('EphysIngest().make(): skip - behavior ingest error')
return
log.info('behavior for ephys: {b}'.format(b=behavior))
#
# Find Ephys Recording
#
key = (experiment.Session & key).fetch1()
sinfo = ((lab.WaterRestriction() * lab.Subject().proj() *
experiment.Session()) & key).fetch1()
rigpath = EphysDataPath().fetch1('data_path')
h2o = sinfo['water_restriction_number']
date = key['session_date'].strftime('%Y%m%d')
dpath = pathlib.Path(rigpath, h2o, date)
dglob = '[0-9]/{}' # probe directory pattern
v3spec = '{}_*_jrc.mat'.format(h2o)
# old v3spec = '{}_g0_*.imec.ap_imec3_opt3_jrc.mat'.format(h2o)
v3files = list(dpath.glob(dglob.format(v3spec)))
v4spec = '{}_*.ap_res.mat'.format(h2o)
# old v4spec = '{}_g0_*.imec?.ap_res.mat'.format(h2o) # TODO v4ify
v4files = list(dpath.glob(dglob.format(v4spec)))
if (v3files and v4files) or not (v3files or v4files):
log.warning(
'Error - v3files ({}) + v4files ({}). Skipping.'.format(
v3files, v4files))
return
if v3files:
files = v3files
loader = self._load_v3
if v4files:
files = v4files
loader = self._load_v4
for f in files:
self._load(loader(sinfo, rigpath, dpath, f.relative_to(dpath)))
def mockdata():
print('populating with mock data')
reload(ccf)
reload(lab)
reload(experiment)
reload(ephys)
reload(publication)
try:
# TODO: these should be loaded in a more 'official' way
lab.Person().insert1({
'username': '******',
'fullname': 'Dave Liu'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification': 'VGAT-Chr2-EYFP Jax',
'gene_modification_description': 'VGAT'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification': 'PV-ires-Cre X Ai32',
'gene_modification_description': 'PV'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification':
'Rosa26 Cag lsl reachR-citrine 1A4 X PV-ires-Cre',
'gene_modification_description': 'reachR PV'
},
skip_duplicates=True)
# Subject 399752 / dl7
lab.Subject().insert1(
{
'subject_id': 399752,
'username': '******',
'cage_number': 145375,
'date_of_birth': '2017-08-03',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 399752,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 399752,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-03',
'end_time': '2017-11-03',
'surgery_description': 'Headbar anterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 399752,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -4,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 399752,
'water_restriction_number': 'dl7',
'cage_number': 148861,
'wr_start_date': '2017-11-07',
'wr_start_weight': 25
},
skip_duplicates=True)
# Subject 397853 / dl14
lab.Subject().insert1(
{
'subject_id': 397853,
'username': '******',
'cage_number': 144545,
'date_of_birth': '2017-07-15',
'sex': 'M',
'animal_source': 'Allen Institute'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 397853,
'gene_modification': 'PV-ires-Cre X Ai32'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 397853,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-20',
'end_time': '2017-11-20',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 397853,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 397853,
'water_restriction_number': 'dl14',
'cage_number': 149595,
'wr_start_date': '2017-11-27',
'wr_start_weight': 24.1
},
skip_duplicates=True)
# Subject 400480 / dl15
lab.Subject().insert1(
{
'subject_id': 400480,
'username': '******',
'cage_number': 145700,
'date_of_birth': '2017-08-09',
'sex': 'M',
'animal_source': 'Allen Institute'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 400480,
'gene_modification': 'PV-ires-Cre X Ai32'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 400480,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-21',
'end_time': '2017-11-21',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 400480,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 400480,
'water_restriction_number': 'dl15',
'cage_number': 149598,
'wr_start_date': '2017-11-27',
'wr_start_weight': 27.6
},
skip_duplicates=True)
# Subject 406680 / dl20
lab.Subject().insert1(
{
'subject_id': 406680,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-06',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 406680,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 406680,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-04',
'end_time': '2018-01-04',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 406680,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 406680,
'water_restriction_number': 'dl20',
'cage_number': 151282,
'wr_start_date': '2018-01-10',
'wr_start_weight': 22.7
},
skip_duplicates=True)
# Subject 408022 / dl21
lab.Subject().insert1(
{
'subject_id': 408022,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-19',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 408022,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 408022,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-05',
'end_time': '2018-01-05',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 408022,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 408022,
'water_restriction_number': 'dl21',
'cage_number': 151283,
'wr_start_date': '2018-01-10',
'wr_start_weight': 21.1
},
skip_duplicates=True)
# Subject 408021 / dl22
lab.Subject().insert1(
{
'subject_id': 408021,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-19',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 408021,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 408021,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-15',
'end_time': '2018-01-15',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 408021,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 408021,
'water_restriction_number': 'dl22',
'cage_number': 151704,
'wr_start_date': '2018-01-19',
'wr_start_weight': 21
},
skip_duplicates=True)
# Subject 407512 / dl24
lab.Subject().insert1(
{
'subject_id': 407512,
'username': '******',
'cage_number': 151629,
'date_of_birth': '2017-10-13',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407512,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407512,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-16',
'end_time': '2018-01-16',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407512,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407512,
'water_restriction_number': 'dl24',
'cage_number': 151793,
'wr_start_date': '2018-01-22',
'wr_start_weight': 26
},
skip_duplicates=True)
# 407513 / dl25
lab.Subject().insert1(
{
'subject_id': 407513,
'username': '******',
'cage_number': 148636,
'date_of_birth': '2017-10-13',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407513,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407513,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-17',
'end_time': '2018-01-17',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407513,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407513,
'water_restriction_number': 'dl25',
'cage_number': 151794,
'wr_start_date': '2018-01-22',
'wr_start_weight': 25.5
},
skip_duplicates=True)
# Subject 407986 / dl28
lab.Subject().insert1(
{
'subject_id': 407986,
'username': '******',
'cage_number': 152268,
'date_of_birth': '2017-10-18',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407986,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407986,
'surgery_id': 1,
'username': '******',
'start_time': '2018-02-01',
'end_time': '2018-02-01',
'surgery_description': 'Headbar anterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407986,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -4,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407986,
'water_restriction_number': 'dl28',
'cage_number': 152312,
'wr_start_date': '2018-02-05',
'wr_start_weight': 19.8
},
skip_duplicates=True)
# Subject 123457 / tw5
lab.Subject().insert1(
{
'subject_id': 123457,
'username': '******',
'cage_number': 145375,
'date_of_birth': '2017-08-03',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 123457,
'water_restriction_number': 'tw5',
'cage_number': 148861,
'wr_start_date': '2017-11-07',
'wr_start_weight': 20.5
},
skip_duplicates=True)
# Subject 412330 / tw34
lab.Subject().insert1(
{
'subject_id': 412330,
'username': '******',
'cage_number': 154522,
'date_of_birth': '2017-12-05',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 412330,
'water_restriction_number': 'tw34',
'cage_number': 154522,
'wr_start_date': '2018-03-18',
'wr_start_weight': 21.0
},
skip_duplicates=True)
# subject 432998 / dl55
lab.Subject().insert1(
{
'subject_id': 432998,
'username': '******',
'cage_number': 160920,
'date_of_birth': '2018-07-02',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 432998,
'water_restriction_number': 'dl55',
'cage_number': 160920,
'wr_start_date': '2018-09-05',
'wr_start_weight': 21.0
},
skip_duplicates=True)
# Subject 435884 / dl59
lab.Subject().insert1(
{
'subject_id': 435884,
'username': '******',
'cage_number': 161908,
'date_of_birth': '2018-08-06',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 435884,
'water_restriction_number': 'dl59',
'cage_number': 154522,
'wr_start_date': '2018-09-30',
'wr_start_weight': 21.0
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig1',
'room': '2w.334',
'rig_description': 'Training rig 1'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig2',
'room': '2w.334',
'rig_description': 'Training rig 2'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig3',
'room': '2w.334',
'rig_description': 'Training rig 3'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'RRig',
'room': '2w.334',
'rig_description': 'Recording rig'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'Ephys1',
'room': '2w.334',
'rig_description': 'Recording computer'
},
skip_duplicates=True)
# ---- Brain Location ----
experiment.BrainLocation.insert1(
{
'brain_location_name': 'left_alm',
'brain_area': 'ALM',
'hemisphere': 'left',
'skull_reference': 'Bregma'
},
skip_duplicates=True)
experiment.BrainLocation.insert1(
{
'brain_location_name': 'right_alm',
'brain_area': 'ALM',
'hemisphere': 'right',
'skull_reference': 'Bregma'
},
skip_duplicates=True)
experiment.BrainLocation.insert1(
{
'brain_location_name': 'both_alm',
'brain_area': 'ALM',
'hemisphere': 'both',
'skull_reference': 'Bregma'
},
skip_duplicates=True)
experiment.BrainLocation.insert1(
{
'brain_location_name': 'left_medulla',
'brain_area': 'Medulla',
'hemisphere': 'left',
'skull_reference': 'Bregma'
},
skip_duplicates=True)
experiment.BrainLocation.insert1(
{
'brain_location_name': 'right_medulla',
'brain_area': 'Medulla',
'hemisphere': 'right',
'skull_reference': 'Bregma'
},
skip_duplicates=True)
experiment.BrainLocation.insert1(
{
'brain_location_name': 'both_medulla',
'brain_area': 'Medulla',
'hemisphere': 'both',
'skull_reference': 'Bregma'
},
skip_duplicates=True)
# Probe (Neuropixel)
npx_probe_model = '15131808323' # using Model No. - SN TBD?
lab.Probe.insert1(
{
'probe': npx_probe_model,
'probe_type': 'neuropixel'
},
skip_duplicates=True,
)
lab.Probe.Electrode.insert(
({
'probe': npx_probe_model,
'electrode': x
} for x in range(1, 961)),
skip_duplicates=True,
)
except Exception as e:
print("error creating mock data: {e}".format(e=e), file=sys.stderr)
raise
def load_meta_foraging():
'''
Load metadata for the foraging task
Adapted from Marton's code: https://github.com/rozmar/DataPipeline/blob/master/ingest/datapipeline_metadata.py
'''
import pathlib
meta_dir = dj.config.get('custom', {}).get('behavior_bpod',
[]).get('meta_dir')
meta_lab_dir = dj.config.get('custom', {}).get('behavior_bpod',
[]).get('meta_lab_dir')
# --- Add experimenters ---
print('Adding experimenters...')
df_experimenters = pd.read_csv(
pathlib.Path(meta_lab_dir) / 'Experimenter.csv')
duplicate_num = 0
for experimenter in df_experimenters.iterrows():
experimenter = experimenter[1]
experimenternow = {
'username': experimenter['username'],
'fullname': experimenter['fullname']
}
try:
lab.Person().insert1(experimenternow)
print(' added experimenter: ', experimenternow['username'])
except dj.errors.DuplicateError:
duplicate_num += 1
# print(' duplicate. experimenter: ',experimenternow['username'], ' already exists')
print(f' {duplicate_num} experimenters already exist')
# --- Add rigs ---
print('Adding rigs... ')
df_rigs = pd.read_csv(pathlib.Path(meta_lab_dir) / 'Rig.csv')
duplicate_num = 0
for rig in df_rigs.iterrows():
rig = rig[1]
rignow = {
'rig': rig['rig'],
'room': rig['room'],
'rig_description': rig['rig_description']
}
try:
lab.Rig().insert1(rignow)
print(' added rig: ', rignow['rig'])
except dj.errors.DuplicateError:
duplicate_num += 1
# print(' duplicate. rig: ',rignow['rig'], ' already exists')
print(f' {duplicate_num} rigs already exist')
# --- Add viruses ---
# Not implemented for now. Han
# --- Add subjects and water restrictions ---
print('Adding subjects and water restrictions...')
df_surgery = pd.read_csv(pathlib.Path(meta_dir) / 'Surgery.csv')
# For each entry
duplicate_subject_num = 0
duplicate_WR_num = 0
for item in df_surgery.iterrows():
item = item[1]
if item['project'] == 'foraging' and (item['status'] == 'training' or
item['status'] == 'sacrificed'):
# -- Add lab.Subject() --
subjectdata = {
'subject_id': item['animal#'],
'cage_number': item['cage#'],
'date_of_birth': item['DOB'],
'sex': item['sex'],
'username': item['experimenter'],
}
try:
lab.Subject().insert1(subjectdata)
print(' added subject: ', item['animal#'])
except dj.errors.DuplicateError:
duplicate_subject_num += 1
# print(' duplicate. animal :',item['animal#'], ' already exists')
# -- Add lab.Surgery() --
# Not implemented. Han
# -- Virus injection --
# Not implemented. Han
# -- Add lab.WaterRestriction() --
if item['ID']:
# Get water restriction start date and weight
subject_csv = pathlib.Path(meta_dir) / '{}.csv'.format(
item['ID'])
if subject_csv.exists():
df_wr = pd.read_csv(subject_csv)
else:
print(' No metadata csv found for {}'.format(item['ID']))
continue
wrdata = {
'subject_id': item['animal#'],
'water_restriction_number': item['ID'],
'cage_number': item['cage#'],
'wr_start_date': df_wr['Date'][0],
'wr_start_weight': df_wr['Weight'][0],
}
try:
lab.WaterRestriction().insert1(wrdata)
print(' added WR: ', item['ID'])
except dj.errors.DuplicateError:
duplicate_WR_num += 1
# print(' duplicate. water restriction:', item['ID'], ' already exists')
print(
f' {duplicate_subject_num} subjects and {duplicate_WR_num} WRs already exist'
)
def populatemetadata():
#%% save metadata from google drive if necessairy
lastmodify = online_notebook.fetch_lastmodify_time_animal_metadata()
with open(dj.config['locations.metadata_surgery_experiment'] +
'last_modify_time.json') as timedata:
lastmodify_prev = json.loads(timedata.read())
if lastmodify != lastmodify_prev:
print('updating surgery and WR metadata from google drive')
dj.config['locations.metadata_surgery_experiment']
df_surgery = online_notebook.fetch_animal_metadata()
df_surgery.to_csv(dj.config['locations.metadata_surgery_experiment'] +
'Surgery.csv')
IDs = df_surgery['ID'].tolist()
for ID in IDs:
df_wr = online_notebook.fetch_water_restriction_metadata(ID)
if type(df_wr) == pd.DataFrame:
df_wr.to_csv(
dj.config['locations.metadata_surgery_experiment'] + ID +
'.csv')
with open(
dj.config['locations.metadata_surgery_experiment'] +
'last_modify_time.json', "w") as write_file:
json.dump(lastmodify, write_file)
print('surgery and WR metadata updated')
lastmodify = online_notebook.fetch_lastmodify_time_lab_metadata()
with open(dj.config['locations.metadata_lab'] +
'last_modify_time.json') as timedata:
lastmodify_prev = json.loads(timedata.read())
if lastmodify != lastmodify_prev:
print('updating Lab metadata from google drive')
dj.config['locations.metadata_lab']
IDs = ['Experimenter', 'Rig', 'Virus']
for ID in IDs:
df_wr = online_notebook.fetch_lab_metadata(ID)
if type(df_wr) == pd.DataFrame:
df_wr.to_csv(dj.config['locations.metadata_lab'] + ID + '.csv')
with open(
dj.config['locations.metadata_lab'] + 'last_modify_time.json',
"w") as write_file:
json.dump(lastmodify, write_file)
print('Lab metadata updated')
#%% add users
df_experimenters = pd.read_csv(dj.config['locations.metadata_lab'] +
'Experimenter.csv')
experimenterdata = list()
for experimenter in df_experimenters.iterrows():
experimenter = experimenter[1]
dictnow = {
'username': experimenter['username'],
'fullname': experimenter['fullname']
}
experimenterdata.append(dictnow)
print('adding experimenters')
for experimenternow in experimenterdata:
try:
lab.Person().insert1(experimenternow)
except dj.errors.DuplicateError:
print('duplicate. experimenter: ', experimenternow['username'],
' already exists')
#%% add rigs
df_rigs = pd.read_csv(dj.config['locations.metadata_lab'] + 'Rig.csv')
rigdata = list()
for rig in df_rigs.iterrows():
rig = rig[1]
dictnow = {
'rig': rig['rig'],
'room': rig['room'],
'rig_description': rig['rig_description']
}
rigdata.append(dictnow)
print('adding rigs')
for rignow in rigdata:
try:
lab.Rig().insert1(rignow)
except dj.errors.DuplicateError:
print('duplicate. rig: ', rignow['rig'], ' already exists')
#%% add viruses
df_viruses = pd.read_csv(dj.config['locations.metadata_lab'] + 'Virus.csv')
virusdata = list()
serotypedata = list()
for virus in df_viruses.iterrows():
virus = virus[1]
if type(virus['remarks']) != str:
virus['remarks'] = ''
dictnow = {
'virus_id': virus['virus_id'],
'virus_source': virus['virus_source'],
'serotype': virus['serotype'],
'username': virus['username'],
'virus_name': virus['virus_name'],
'titer': virus['titer'],
'order_date': virus['order_date'],
'remarks': virus['remarks']
}
virusdata.append(dictnow)
dictnow = {'serotype': virus['serotype']}
serotypedata.append(dictnow)
print('adding rigs')
for virusnow, serotypenow in zip(virusdata, serotypedata):
try:
lab.Serotype().insert1(serotypenow)
except dj.errors.DuplicateError:
print('duplicate serotype: ', serotypenow['serotype'],
' already exists')
try:
lab.Virus().insert1(virusnow)
except dj.errors.DuplicateError:
print('duplicate virus: ', virusnow['virus_name'],
' already exists')
#%% populate subjects, surgeries and water restrictions
print('adding surgeries and stuff')
df_surgery = pd.read_csv(
dj.config['locations.metadata_surgery_experiment'] + 'Surgery.csv')
#%%
for item in df_surgery.iterrows():
if item[1]['project'] == dj.config['project'] and (
item[1]['status'] == 'training'
or item[1]['status'] == 'sacrificed'):
subjectdata = {
'subject_id': item[1]['animal#'],
'cage_number': item[1]['cage#'],
'date_of_birth': item[1]['DOB'],
'sex': item[1]['sex'],
'username': item[1]['experimenter'],
}
try:
lab.Subject().insert1(subjectdata)
except dj.errors.DuplicateError:
print('duplicate. animal :', item[1]['animal#'],
' already exists')
surgeryidx = 1
while 'surgery date (' + str(surgeryidx) + ')' in item[1].keys(
) and item[1]['surgery date (' + str(surgeryidx) + ')'] and type(
item[1]['surgery date (' + str(surgeryidx) + ')']) == str:
start_time = datetime.strptime(
item[1]['surgery date (' + str(surgeryidx) + ')'] + ' ' +
item[1]['surgery time (' + str(surgeryidx) + ')'],
'%Y-%m-%d %H:%M')
end_time = start_time + timedelta(
minutes=int(item[1]['surgery length (min) (' +
str(surgeryidx) + ')']))
surgerydata = {
'surgery_id':
surgeryidx,
'subject_id':
item[1]['animal#'],
'username':
item[1]['experimenter'],
'start_time':
start_time,
'end_time':
end_time,
'surgery_description':
item[1]['surgery type (' + str(surgeryidx) + ')'] +
':-: comments: ' +
str(item[1]['surgery comments (' + str(surgeryidx) + ')']),
}
try:
lab.Surgery().insert1(surgerydata)
except dj.errors.DuplicateError:
print('duplicate. surgery for animal ', item[1]['animal#'],
' already exists: ', start_time)
#checking craniotomies
#%
cranioidx = 1
while 'craniotomy diameter (' + str(
cranioidx) + ')' in item[1].keys() and item[1][
'craniotomy diameter (' + str(cranioidx) +
')'] and (
type(item[1]['craniotomy surgery id (' +
str(cranioidx) + ')']) == int or
type(item[1]['craniotomy surgery id (' +
str(cranioidx) + ')']) == float):
if item[1]['craniotomy surgery id (' + str(cranioidx) +
')'] == surgeryidx:
proceduredata = {
'surgery_id':
surgeryidx,
'subject_id':
item[1]['animal#'],
'procedure_id':
cranioidx,
'skull_reference':
item[1]['craniotomy reference (' + str(cranioidx) +
')'],
'ml_location':
item[1]['craniotomy lateral (' + str(cranioidx) +
')'],
'ap_location':
item[1]['craniotomy anterior (' + str(cranioidx) +
')'],
'surgery_procedure_description':
'craniotomy: ' + item[1]['craniotomy comments (' +
str(cranioidx) + ')'],
}
try:
lab.Surgery.Procedure().insert1(proceduredata)
except dj.errors.DuplicateError:
print('duplicate cranio for animal ',
item[1]['animal#'], ' already exists: ',
cranioidx)
cranioidx += 1
#%
virusinjidx = 1
while 'virus inj surgery id (' + str(
virusinjidx) + ')' in item[1].keys() and item[1][
'virus inj virus id (' + str(virusinjidx) +
')'] and item[1]['virus inj surgery id (' +
str(virusinjidx) + ')']:
if item[1]['virus inj surgery id (' + str(virusinjidx) +
')'] == surgeryidx:
# =============================================================================
# print('waiting')
# timer.sleep(1000)
# =============================================================================
if '[' in item[1]['virus inj lateral (' +
str(virusinjidx) + ')']:
virus_ml_locations = eval(
item[1]['virus inj lateral (' +
str(virusinjidx) + ')'])
virus_ap_locations = eval(
item[1]['virus inj anterior (' +
str(virusinjidx) + ')'])
virus_dv_locations = eval(
item[1]['virus inj ventral (' +
str(virusinjidx) + ')'])
virus_volumes = eval(
item[1]['virus inj volume (nl) (' +
str(virusinjidx) + ')'])
else:
virus_ml_locations = [
int(item[1]['virus inj lateral (' +
str(virusinjidx) + ')'])
]
virus_ap_locations = [
int(item[1]['virus inj anterior (' +
str(virusinjidx) + ')'])
]
virus_dv_locations = [
int(item[1]['virus inj ventral (' +
str(virusinjidx) + ')'])
]
virus_volumes = [
int(item[1]['virus inj volume (nl) (' +
str(virusinjidx) + ')'])
]
for virus_ml_location, virus_ap_location, virus_dv_location, virus_volume in zip(
virus_ml_locations, virus_ap_locations,
virus_dv_locations, virus_volumes):
injidx = len(lab.Surgery.VirusInjection()
& surgerydata) + 1
virusinjdata = {
'surgery_id':
surgeryidx,
'subject_id':
item[1]['animal#'],
'injection_id':
injidx,
'virus_id':
item[1]['virus inj virus id (' +
str(virusinjidx) + ')'],
'skull_reference':
item[1]['virus inj reference (' +
str(virusinjidx) + ')'],
'ml_location':
virus_ml_location,
'ap_location':
virus_ap_location,
'dv_location':
virus_dv_location,
'volume':
virus_volume,
'dilution':
item[1]['virus inj dilution (' +
str(virusinjidx) + ')'],
'description':
'virus injection: ' +
item[1]['virus inj comments (' +
str(virusinjidx) + ')'],
}
try:
lab.Surgery.VirusInjection().insert1(
virusinjdata)
except dj.errors.DuplicateError:
print('duplicate virus injection for animal ',
item[1]['animal#'], ' already exists: ',
injidx)
virusinjidx += 1
#%
surgeryidx += 1
#%
if item[1]['ID']:
#df_wr = online_notebook.fetch_water_restriction_metadata(item[1]['ID'])
try:
df_wr = pd.read_csv(
dj.config['locations.metadata_surgery_experiment'] +
item[1]['ID'] + '.csv')
except:
df_wr = None
if type(df_wr) == pd.DataFrame:
wrdata = {
'subject_id': item[1]['animal#'],
'water_restriction_number': item[1]['ID'],
'cage_number': item[1]['cage#'],
'wr_start_date': df_wr['Date'][0],
'wr_start_weight': df_wr['Weight'][0],
}
try:
lab.WaterRestriction().insert1(wrdata)
except dj.errors.DuplicateError:
print('duplicate. water restriction :',
item[1]['animal#'], ' already exists')
for subject_now in [
'leaky3t.05c30h', 'leaky3t.05c15h', 'leaky3t.05c5h', 'W-St-L-Rnd',
'W-St-L-Sw', 'leaky3t5it30h', 'leaky3t3it30h', 'cheater'
]:
#subject_now = 'leaky3ms.05c30t'#'leaky3ms.05c15t'#'leaky3ms.05c30t'#'W-St-L-Rnd'#'W-St-L-Sw' #'leakyint_9ms+c'#'leakyint_3ms+c'#'
setupname = 'virtual_setup'
experimenter = 'rozsam'
print('subject: ', subject_now)
if len(lab.WaterRestriction()
& 'water_restriction_number = "' + subject_now + '"') > 0:
subject_id_to_del = (lab.WaterRestriction()
& 'water_restriction_number = "' + subject_now +
'"').fetch('subject_id')[0]
dj.config['safemode'] = False
(lab.Subject() & 'subject_id = ' + str(subject_id_to_del)).delete()
dj.config['safemode'] = True
for subject_id in range(100):
if len(lab.Subject & 'subject_id = ' + str(subject_id)) == 0:
break
#%
subjectdata = {
'subject_id': subject_id,
'cage_number': 0,
'date_of_birth': datetime.now().strftime('%Y-%m-%d'),
'sex': 'Unknown',
'username': experimenter,
}
lab.Subject().insert1(subjectdata)
wrdata = {
'subject_id': subject_id,
def mockdata():
print('populating with mock data')
reload(ccf)
reload(lab)
reload(experiment)
reload(ephys)
reload(publication)
try:
# TODO: these should be loaded in a more 'official' way
lab.Person().insert1({
'username': '******',
'fullname': 'Dave Liu'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification': 'VGAT-Chr2-EYFP Jax',
'gene_modification_description': 'VGAT'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification': 'PV-ires-Cre X Ai32',
'gene_modification_description': 'PV'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification':
'Rosa26 Cag lsl reachR-citrine 1A4 X PV-ires-Cre',
'gene_modification_description': 'reachR PV'
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 399752,
'username': '******',
'cage_number': 145375,
'date_of_birth': '2017-08-03',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 399752,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 399752,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-03',
'end_time': '2017-11-03',
'surgery_description': 'Headbar anterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 399752,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -4,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 399752,
'water_restriction_number': 'dl7',
'cage_number': 148861,
'wr_start_date': '2017-11-07',
'wr_start_weight': 25
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 397853,
'username': '******',
'cage_number': 144545,
'date_of_birth': '2017-07-15',
'sex': 'M',
'animal_source': 'Allen Institute'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 397853,
'gene_modification': 'PV-ires-Cre X Ai32'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 397853,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-20',
'end_time': '2017-11-20',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 397853,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 397853,
'water_restriction_number': 'dl14',
'cage_number': 149595,
'wr_start_date': '2017-11-27',
'wr_start_weight': 24.1
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 400480,
'username': '******',
'cage_number': 145700,
'date_of_birth': '2017-08-09',
'sex': 'M',
'animal_source': 'Allen Institute'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 400480,
'gene_modification': 'PV-ires-Cre X Ai32'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 400480,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-21',
'end_time': '2017-11-21',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 400480,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 400480,
'water_restriction_number': 'dl15',
'cage_number': 149598,
'wr_start_date': '2017-11-27',
'wr_start_weight': 27.6
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 406680,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-06',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 406680,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 406680,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-04',
'end_time': '2018-01-04',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 406680,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 406680,
'water_restriction_number': 'dl20',
'cage_number': 151282,
'wr_start_date': '2018-01-10',
'wr_start_weight': 22.7
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 408022,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-19',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 408022,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 408022,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-05',
'end_time': '2018-01-05',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 408022,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 408022,
'water_restriction_number': 'dl21',
'cage_number': 151283,
'wr_start_date': '2018-01-10',
'wr_start_weight': 21.1
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 408021,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-19',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 408021,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 408021,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-15',
'end_time': '2018-01-15',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 408021,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 408021,
'water_restriction_number': 'dl22',
'cage_number': 151704,
'wr_start_date': '2018-01-19',
'wr_start_weight': 21
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 407512,
'username': '******',
'cage_number': 151629,
'date_of_birth': '2017-10-13',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407512,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407512,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-16',
'end_time': '2018-01-16',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407512,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407512,
'water_restriction_number': 'dl24',
'cage_number': 151793,
'wr_start_date': '2018-01-22',
'wr_start_weight': 26
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 407513,
'username': '******',
'cage_number': 148636,
'date_of_birth': '2017-10-13',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407513,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407513,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-17',
'end_time': '2018-01-17',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407513,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407513,
'water_restriction_number': 'dl25',
'cage_number': 151794,
'wr_start_date': '2018-01-22',
'wr_start_weight': 25.5
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 407986,
'username': '******',
'cage_number': 152268,
'date_of_birth': '2017-10-18',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407986,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407986,
'surgery_id': 1,
'username': '******',
'start_time': '2018-02-01',
'end_time': '2018-02-01',
'surgery_description': 'Headbar anterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407986,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -4,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407986,
'water_restriction_number': 'dl28',
'cage_number': 152312,
'wr_start_date': '2018-02-05',
'wr_start_weight': 19.8
},
skip_duplicates=True)
lab.Subject().insert1(
{
'subject_id': 123457,
'username': '******',
'cage_number': 145375,
'date_of_birth': '2017-08-03',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 123457,
'water_restriction_number': 'tw5',
'cage_number': 148861,
'wr_start_date': '2017-11-07',
'wr_start_weight': 20.5
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig1',
'room': '2w.334',
'rig_description': 'Training rig 1'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig2',
'room': '2w.334',
'rig_description': 'Training rig 2'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig3',
'room': '2w.334',
'rig_description': 'Training rig 3'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'RRig',
'room': '2w.334',
'rig_description': 'Recording rig'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'Ephys1',
'room': '2w.334',
'rig_description': 'Recording computer'
},
skip_duplicates=True)
except Exception as e:
print("error creating mock data: {e}".format(e=e), file=sys.stderr)
raise
def mockdata():
print('populating with mock data')
reload(lab)
reload(ccf)
reload(experiment)
reload(ephys)
reload(histology)
reload(tracking)
try:
lab.Person().insert1({
'username': '******',
'fullname': 'Unknown'
},
skip_duplicates=True)
lab.Person().insert1({
'username': '******',
'fullname': 'Dave Liu'
},
skip_duplicates=True)
lab.Person().insert1({
'username': '******',
'fullname': 'Susu Chen'
},
skip_duplicates=True)
lab.Person().insert1({
'username': '******',
'fullname': 'Thinh Nguyen'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification': 'VGAT-Chr2-EYFP Jax',
'gene_modification_description': 'VGAT'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification': 'PV-ires-Cre X Ai32',
'gene_modification_description': 'PV'
},
skip_duplicates=True)
lab.ModifiedGene().insert1(
{
'gene_modification':
'Rosa26 Cag lsl reachR-citrine 1A4 X PV-ires-Cre',
'gene_modification_description': 'reachR PV'
},
skip_duplicates=True)
# Subject 399752 / dl7
lab.Subject().insert1(
{
'subject_id': 399752,
'username': '******',
'cage_number': 145375,
'date_of_birth': '2017-08-03',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 399752,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 399752,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-03',
'end_time': '2017-11-03',
'surgery_description': 'Headbar anterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 399752,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -4,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 399752,
'water_restriction_number': 'dl7',
'cage_number': 148861,
'wr_start_date': '2017-11-07',
'wr_start_weight': 25
},
skip_duplicates=True)
# Subject 397853 / dl14
lab.Subject().insert1(
{
'subject_id': 397853,
'username': '******',
'cage_number': 144545,
'date_of_birth': '2017-07-15',
'sex': 'M',
'animal_source': 'Allen Institute'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 397853,
'gene_modification': 'PV-ires-Cre X Ai32'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 397853,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-20',
'end_time': '2017-11-20',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 397853,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 397853,
'water_restriction_number': 'dl14',
'cage_number': 149595,
'wr_start_date': '2017-11-27',
'wr_start_weight': 24.1
},
skip_duplicates=True)
# Subject 400480 / dl15
lab.Subject().insert1(
{
'subject_id': 400480,
'username': '******',
'cage_number': 145700,
'date_of_birth': '2017-08-09',
'sex': 'M',
'animal_source': 'Allen Institute'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 400480,
'gene_modification': 'PV-ires-Cre X Ai32'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 400480,
'surgery_id': 1,
'username': '******',
'start_time': '2017-11-21',
'end_time': '2017-11-21',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 400480,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 400480,
'water_restriction_number': 'dl15',
'cage_number': 149598,
'wr_start_date': '2017-11-27',
'wr_start_weight': 27.6
},
skip_duplicates=True)
# Subject 406680 / dl20
lab.Subject().insert1(
{
'subject_id': 406680,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-06',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 406680,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 406680,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-04',
'end_time': '2018-01-04',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 406680,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 406680,
'water_restriction_number': 'dl20',
'cage_number': 151282,
'wr_start_date': '2018-01-10',
'wr_start_weight': 22.7
},
skip_duplicates=True)
# Subject 408022 / dl21
lab.Subject().insert1(
{
'subject_id': 408022,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-19',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 408022,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 408022,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-05',
'end_time': '2018-01-05',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 408022,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 408022,
'water_restriction_number': 'dl21',
'cage_number': 151283,
'wr_start_date': '2018-01-10',
'wr_start_weight': 21.1
},
skip_duplicates=True)
# Subject 408021 / dl22
lab.Subject().insert1(
{
'subject_id': 408021,
'username': '******',
'cage_number': 148859,
'date_of_birth': '2017-10-19',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 408021,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 408021,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-15',
'end_time': '2018-01-15',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 408021,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 408021,
'water_restriction_number': 'dl22',
'cage_number': 151704,
'wr_start_date': '2018-01-19',
'wr_start_weight': 21
},
skip_duplicates=True)
# Subject 407512 / dl24
lab.Subject().insert1(
{
'subject_id': 407512,
'username': '******',
'cage_number': 151629,
'date_of_birth': '2017-10-13',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407512,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407512,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-16',
'end_time': '2018-01-16',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407512,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407512,
'water_restriction_number': 'dl24',
'cage_number': 151793,
'wr_start_date': '2018-01-22',
'wr_start_weight': 26
},
skip_duplicates=True)
# 407513 / dl25
lab.Subject().insert1(
{
'subject_id': 407513,
'username': '******',
'cage_number': 148636,
'date_of_birth': '2017-10-13',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407513,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407513,
'surgery_id': 1,
'username': '******',
'start_time': '2018-01-17',
'end_time': '2018-01-17',
'surgery_description': 'Headbar posterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407513,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -1.75,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407513,
'water_restriction_number': 'dl25',
'cage_number': 151794,
'wr_start_date': '2018-01-22',
'wr_start_weight': 25.5
},
skip_duplicates=True)
# Subject 407986 / dl28
lab.Subject().insert1(
{
'subject_id': 407986,
'username': '******',
'cage_number': 152268,
'date_of_birth': '2017-10-18',
'sex': 'F',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.Subject.GeneModification().insert1(
{
'subject_id': 407986,
'gene_modification': 'VGAT-Chr2-EYFP Jax'
},
skip_duplicates=True)
lab.Surgery().insert1(
{
'subject_id': 407986,
'surgery_id': 1,
'username': '******',
'start_time': '2018-02-01',
'end_time': '2018-02-01',
'surgery_description': 'Headbar anterior'
},
skip_duplicates=True)
lab.Surgery.Procedure().insert1(
{
'subject_id': 407986,
'surgery_id': 1,
'procedure_id': 1,
'skull_reference': 'Bregma',
'ml_location': 0,
'ap_location': -4,
'surgery_procedure_description': 'Fiducial marker'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 407986,
'water_restriction_number': 'dl28',
'cage_number': 152312,
'wr_start_date': '2018-02-05',
'wr_start_weight': 19.8
},
skip_duplicates=True)
# Subject 123457 / tw5
lab.Subject().insert1(
{
'subject_id': 123457,
'username': '******',
'cage_number': 145375,
'date_of_birth': '2017-08-03',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 123457,
'water_restriction_number': 'tw5',
'cage_number': 148861,
'wr_start_date': '2017-11-07',
'wr_start_weight': 20.5
},
skip_duplicates=True)
# Subject 412330 / tw34
lab.Subject().insert1(
{
'subject_id': 412330,
'username': '******',
'cage_number': 154522,
'date_of_birth': '2017-12-05',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 412330,
'water_restriction_number': 'tw34',
'cage_number': 154522,
'wr_start_date': '2018-03-18',
'wr_start_weight': 21.0
},
skip_duplicates=True)
# subject 432998 / dl55
lab.Subject().insert1(
{
'subject_id': 432998,
'username': '******',
'cage_number': 160920,
'date_of_birth': '2018-07-02',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 432998,
'water_restriction_number': 'dl55',
'cage_number': 160920,
'wr_start_date': '2018-09-05',
'wr_start_weight': 21.0
},
skip_duplicates=True)
# Subject 435884 / dl59
lab.Subject().insert1(
{
'subject_id': 435884,
'username': '******',
'cage_number': 161908,
'date_of_birth': '2018-08-06',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 435884,
'water_restriction_number': 'dl59',
'cage_number': 154522,
'wr_start_date': '2018-09-30',
'wr_start_weight': 21.0
},
skip_duplicates=True)
# Subject 432572 / dl56
lab.Subject().insert1(
{
'subject_id': 432572,
'username': '******',
'cage_number': 161125,
'date_of_birth': '2018-06-28',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 432572,
'water_restriction_number': 'dl56',
'cage_number': 161125,
'wr_start_date': '2018-09-10',
'wr_start_weight': 21.0
},
skip_duplicates=True)
# Subject 412753 / dl36
lab.Subject().insert1(
{
'subject_id': 412753,
'username': '******',
'cage_number': 154570,
'date_of_birth': '2017-12-07',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 412753,
'water_restriction_number': 'dl36',
'cage_number': 154570,
'wr_start_date': '2017-03-30',
'wr_start_weight': 21.0
},
skip_duplicates=True)
# Subject 440010 / dl62
lab.Subject().insert1(
{
'subject_id': 440010,
'username': '******',
'cage_number': 163782,
'date_of_birth': '2018-09-24',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 440010,
'water_restriction_number': 'dl62',
'cage_number': 163782,
'wr_start_date': '2018-11-24',
'wr_start_weight': 23.0
},
skip_duplicates=True)
# Subject 55004 / DL004
lab.Subject().insert1(
{
'subject_id': 550004,
'username': '******',
'cage_number': 163782,
'date_of_birth': '2018-09-24',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 550004,
'water_restriction_number': 'DL004',
'cage_number': 163782,
'wr_start_date': '2018-11-24',
'wr_start_weight': 23.0
},
skip_duplicates=True)
# Subject 550009 / DL009
lab.Subject().insert1(
{
'subject_id': 550009,
'username': '******',
'cage_number': 163782,
'date_of_birth': '2018-09-24',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 550009,
'water_restriction_number': 'DL009',
'cage_number': 163782,
'wr_start_date': '2018-11-24',
'wr_start_weight': 23.0
},
skip_duplicates=True)
# Subject 550010 / DL010
lab.Subject().insert1(
{
'subject_id': 550010,
'username': '******',
'cage_number': 163782,
'date_of_birth': '2018-09-24',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 550010,
'water_restriction_number': 'DL010',
'cage_number': 163782,
'wr_start_date': '2018-11-24',
'wr_start_weight': 23.0
},
skip_duplicates=True)
# Subject 550022 / DL022
lab.Subject().insert1(
{
'subject_id': 550022,
'username': '******',
'cage_number': 163782,
'date_of_birth': '2018-09-24',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 550022,
'water_restriction_number': 'DL022',
'cage_number': 163782,
'wr_start_date': '2018-11-24',
'wr_start_weight': 23.0
},
skip_duplicates=True)
# Subject 440959 / SC011
lab.Subject().insert1(
{
'subject_id': 440959,
'username': '******',
'cage_number': 440959,
'date_of_birth': '2018-10-09',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 440959,
'water_restriction_number': 'SC011',
'cage_number': 440959,
'wr_start_date': '2018-12-21',
'wr_start_weight': 22.8
},
skip_duplicates=True)
# Subject 442571 / SC022
lab.Subject().insert1(
{
'subject_id': 442571,
'username': '******',
'cage_number': 442571,
'date_of_birth': '2018-10-29',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 442571,
'water_restriction_number': 'SC022',
'cage_number': 442571,
'wr_start_date': '2019-01-02',
'wr_start_weight': 26.5
},
skip_duplicates=True)
# Subject 460432 / SC030
lab.Subject().insert1(
{
'subject_id': 456773,
'username': '******',
'cage_number': 171857,
'date_of_birth': '2019-05-29',
'sex': 'M',
'animal_source': 'Jackson Labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 456773,
'water_restriction_number': 'SC030',
'cage_number': 171857,
'wr_start_date': '2019-08-08',
'wr_start_weight': 25.700
},
skip_duplicates=True)
# Subject 460432 / SC032
lab.Subject().insert1(
{
'subject_id': 460432,
'username': '******',
'cage_number': 173167,
'date_of_birth': '2019-07-15',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 460432,
'water_restriction_number': 'SC032',
'cage_number': 173167,
'wr_start_date': '2019-09-20',
'wr_start_weight': 22.8
},
skip_duplicates=True)
for num in range(1, 20):
lab.Subject().insert1(
{
'subject_id': 777000 + num,
'username': '******',
'cage_number': 173167,
'date_of_birth': '2019-07-15',
'sex': 'M',
'animal_source': 'Jackson labs'
},
skip_duplicates=True)
lab.WaterRestriction().insert1(
{
'subject_id': 777000 + num,
'water_restriction_number': 'FOR' + f'{num:02}',
'cage_number': 173167,
'wr_start_date': '2019-09-20',
'wr_start_weight': 22.8
},
skip_duplicates=True)
# Rig
lab.Rig().insert1(
{
'rig': 'TRig1',
'room': '2w.334',
'rig_description': 'Training rig 1'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig2',
'room': '2w.334',
'rig_description': 'Training rig 2'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'TRig3',
'room': '2w.334',
'rig_description': 'Training rig 3'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'RRig',
'room': '2w.334',
'rig_description': 'Recording rig'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'RRig2',
'room': '2w.334',
'rig_description': 'Recording rig2'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'RRig3',
'room': '2w.334',
'rig_description': 'Recording rig3 for multi-target-licking'
},
skip_duplicates=True)
lab.Rig().insert1(
{
'rig': 'Ephys1',
'room': '2w.334',
'rig_description': 'Recording computer'
},
skip_duplicates=True)
lab.ProbeType.create_neuropixels_probe()
except Exception as e:
print("error creating mock data: {e}".format(e=e), file=sys.stderr)
raise
def populateelphys():
#%%
df_subject_wr_sessions = pd.DataFrame(lab.WaterRestriction() *
experiment.Session() *
experiment.SessionDetails)
df_subject_ids = pd.DataFrame(lab.Subject())
if len(df_subject_wr_sessions) > 0:
subject_names = df_subject_wr_sessions[
'water_restriction_number'].unique()
subject_names.sort()
else:
subject_names = list()
subject_ids = df_subject_ids['subject_id'].unique()
#%
sumdata = list()
basedir = Path(dj.config['locations.elphysdata_acq4'])
for setup_dir in basedir.iterdir():
setup_name = setup_dir.name
sessions = np.sort(
os.listdir(setup_dir)
) #configfile.readConfigFile(setup_dir.joinpath('.index'))
for session_acq in sessions[::-1]: #.keys():
if session_acq != '.' and session_acq != 'log.txt':
session_dir = setup_dir.joinpath(session_acq)
try:
cells = configfile.readConfigFile(
session_dir.joinpath('.index'))
except: # if there is no file
cells = None
if cells and 'WR_name/ID' in cells['.'].keys(
): # it needs to have WRname
wrname_ephys = cells['.']['WR_name/ID']
wrname = None
for wrname_potential in subject_names: # look for water restriction number
if wrname_potential.lower() in wrname_ephys.lower():
wrname = wrname_potential
subject_id = (df_subject_wr_sessions.loc[
df_subject_wr_sessions[
'water_restriction_number'] == wrname,
'subject_id']).unique()[0]
if wrname == None: # look for animal identifier:
for wrname_potential in subject_ids: # look for water restriction number
if str(wrname_potential) in wrname_ephys.lower():
subject_id = wrname_potential
if len(df_subject_wr_sessions) > 0 and len(
(df_subject_wr_sessions.loc[
df_subject_wr_sessions['subject_id'] ==
subject_id, 'water_restriction_number']
).unique()) > 0:
wrname = (df_subject_wr_sessions.loc[
df_subject_wr_sessions['subject_id'] ==
subject_id, 'water_restriction_number']
).unique()[0]
else:
wrname = 'no water restriction number for this mouse'
if wrname:
session_date = (
session_acq[0:session_acq.find('_')]).replace(
'.', '-')
print('animal: ' + str(subject_id) + ' - ' +
wrname) ##
if setup_name == 'Voltage_rig_1P':
setupname = 'Voltage-Imaging-1p'
else:
print('unkwnown setup, please add')
timer.wait(1000)
if 'experimenter' in cells['.'].keys():
username = cells['.']['experimenter']
else:
username = '******'
print(
'username not specified in acq4 file, assuming rozsam'
)
### check if session already exists
sessiondata = {
'subject_id':
subject_id, #(lab.WaterRestriction() & 'water_restriction_number = "'+df_behavior_session['subject'][0]+'"').fetch()[0]['subject_id'],
'session': np.nan,
'session_date': session_date,
'session_time':
np.nan, #session_time.strftime('%H:%M:%S'),
'username': username,
'rig': setupname
}
for cell in cells.keys():
if cell != '.' and cell != 'log.txt':
ephisdata_cell = list()
sweepstarttimes = list()
cell_dir = session_dir.joinpath(cell)
serieses = configfile.readConfigFile(
cell_dir.joinpath('.index'))
cellstarttime = datetime.datetime.fromtimestamp(
serieses['.']['__timestamp__'])
for series in serieses.keys():
if series != '.' and series != 'log.txt':
series_dir = cell_dir.joinpath(series)
sweeps = configfile.readConfigFile(
series_dir.joinpath('.index'))
if 'Clamp1.ma' in sweeps.keys():
protocoltype = 'single sweep'
sweepkeys = ['']
else:
protocoltype = 'multiple sweeps'
sweepkeys = sweeps.keys()
for sweep in sweepkeys:
if sweep != '.' and '.txt' not in sweep and '.ma' not in sweep:
sweep_dir = series_dir.joinpath(
sweep)
sweepinfo = configfile.readConfigFile(
sweep_dir.joinpath(
'.index'))
if sweep == '':
sweep = '0'
for file in sweepinfo.keys():
if '.ma' in file:
try: # old file version
#print('new file version')
#%
ephysfile = h5.File(
sweep_dir.
joinpath(file),
"r")
data = ephysfile[
'data'][()]
metadata_h5 = ephysfile[
'info']
metadata = read_h5f_metadata(
metadata_h5)
daqchannels = list(
metadata[2]
['DAQ'].keys())
sweepstarttime = datetime.datetime.fromtimestamp(
metadata[2]
['DAQ']
[daqchannels[
0]]
['startTime'])
relativetime = (
sweepstarttime
- cellstarttime
).total_seconds()
if len(
ephisdata_cell
) > 0 and ephisdata_cell[
-1]['sweepstarttime'] == sweepstarttime:
ephisdata = ephisdata_cell.pop(
)
else:
ephisdata = dict(
)
if 'primary' in daqchannels: # ephys data
ephisdata[
'V'] = data[
1]
ephisdata[
'stim'] = data[
0]
ephisdata[
'data'] = data
ephisdata[
'metadata'] = metadata
ephisdata[
'time'] = metadata[
1]['values']
ephisdata[
'relativetime'] = relativetime
ephisdata[
'sweepstarttime'] = sweepstarttime
ephisdata[
'series'] = series
ephisdata[
'sweep'] = sweep
sweepstarttimes.append(
sweepstarttime
)
else: # other daq stuff
#%
for idx, channel in enumerate(
metadata[
0]
['cols']):
channelname = channel[
'name'].decode(
)
if channelname[
0] == 'u':
channelname = channelname[
2:
-1]
if channelname in [
'OrcaFlashExposure',
'Temperature',
'LED525',
'FrameCommand',
'NextFileTrigger'
]:
ephisdata[
channelname] = data[
idx]
#print('{} added'.format(channelname))
else:
print(
'waiting in the other daq'
)
timer.sleep(
1000
)
ephisdata_cell.append(
ephisdata)
#%
except: # new file version
print(
'old version')
ephysfile = MetaArray(
)
ephysfile.readFile(
sweep_dir.
joinpath(file))
data = ephysfile.asarray(
)
metadata = ephysfile.infoCopy(
)
sweepstarttime = datetime.datetime.fromtimestamp(
metadata[2]
['startTime'])
relativetime = (
sweepstarttime
- cellstarttime
).total_seconds()
ephisdata = dict()
ephisdata[
'V'] = data[1]
ephisdata[
'stim'] = data[
0]
ephisdata[
'data'] = data
ephisdata[
'metadata'] = metadata
ephisdata[
'time'] = metadata[
1]['values']
ephisdata[
'relativetime'] = relativetime
ephisdata[
'sweepstarttime'] = sweepstarttime
ephisdata[
'series'] = series
ephisdata[
'sweep'] = sweep
sweepstarttimes.append(
sweepstarttime)
ephisdata_cell.append(
ephisdata)
# ============================================================================
# if wrname == 'FOR04':
# =============================================================================
# add session to DJ if not present
if len(ephisdata_cell) > 0:
# =============================================================================
# print('waiting')
# timer.sleep(1000)
# =============================================================================
#%
if len(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) + '"'
& 'session_date = "' +
str(sessiondata['session_date']) +
'"') == 0:
if len(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) +
'"') == 0:
sessiondata['session'] = 1
else:
sessiondata['session'] = len(
(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id'])
+ '"').fetch()['session']) + 1
sessiondata['session_time'] = (
sweepstarttimes[0]
).strftime(
'%H:%M:%S'
) # the time of the first sweep will be the session time
experiment.Session().insert1(
sessiondata)
#%
session = (
experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) + '"'
& 'session_date = "' +
str(sessiondata['session_date']) +
'"').fetch('session')[0]
cell_number = int(cell[cell.find('_') +
1:])
#add cell if not added already
celldata = {
'subject_id': subject_id,
'session': session,
'cell_number': cell_number,
}
#%
if len(ephys_patch.Cell() & celldata
) == 0 or len(ephys_patch.Cell() *
ephys_patch.Sweep()
& celldata) < len(
ephisdata_cell):
if len(ephys_patch.Cell() *
ephys_patch.Sweep() & celldata
) < len(ephisdata_cell):
print('finishing a recording:')
else:
print('adding new recording:')
print(celldata)
if 'type' in serieses['.'].keys():
if serieses['.'][
'type'] == 'interneuron':
celldata['cell_type'] = 'int'
elif serieses['.'][
'type'] == 'unknown' or serieses[
'.']['type'] == 'fail':
celldata[
'cell_type'] = 'unidentified'
else:
print('unhandled cell type!!')
timer.sleep(1000)
else:
celldata[
'cell_type'] = 'unidentified'
celldata['cell_recording_start'] = (
sweepstarttimes[0]
).strftime('%H:%M:%S')
if 'depth' in serieses['.'].keys(
) and len(serieses['.']['depth']) > 0:
celldata['depth'] = int(
serieses['.']['depth'])
else:
celldata['depth'] = -1
try:
ephys_patch.Cell().insert1(
celldata,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
if 'notes' in serieses['.'].keys():
cellnotes = serieses['.']['notes']
else:
cellnotes = ''
cellnotesdata = {
'subject_id': subject_id,
'session': session,
'cell_number': cell_number,
'notes': cellnotes
}
try:
ephys_patch.CellNotes().insert1(
cellnotesdata,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
#%
for i, ephisdata in enumerate(
ephisdata_cell):
#%
sweep_number = i
print('sweep {}'.format(
sweep_number))
sweep_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'sweep_start_time':
(ephisdata['sweepstarttime'] -
sweepstarttimes[0]
).total_seconds(),
'sweep_end_time':
(ephisdata['sweepstarttime'] -
sweepstarttimes[0]
).total_seconds() +
ephisdata['time'][-1],
'protocol_name':
ephisdata[
'series'], #[:ephisdata['series'].find('_')],
'protocol_sweep_number':
int(ephisdata['sweep'])
}
if 'mode' in ephisdata['metadata'][
2]['ClampState']: # old file version
recmode = ephisdata[
'metadata'][2][
'ClampState']['mode']
else:
recmode = ephisdata[
'metadata'][2]['Protocol'][
'mode']
if 'IC' in str(recmode):
recording_mode = 'current clamp'
else:
print(
'unhandled recording mode, please act..'
)
timer.sleep(10000)
channelnames = list()
channelunits = list()
for line_now in ephisdata[
'metadata'][0]['cols']:
if type(line_now['name']
) == bytes:
channelnames.append(
line_now['name'].
decode().strip("'"))
channelunits.append(
line_now['units'].
decode().strip("'"))
else:
channelnames.append(
line_now['name'])
channelunits.append(
line_now['units'])
commandidx = np.where(
np.array(channelnames) ==
'command')[0][0]
dataidx = np.where(
np.array(channelnames) ==
'primary')[0][0]
#%
clampparams_data = ephisdata[
'metadata'][2]['ClampState'][
'ClampParams'].copy()
clampparams_data_new = dict()
for clampparamkey in clampparams_data.keys(
): #6004 is true for some reason.. changing it back to 1
if type(clampparams_data[
clampparamkey]
) == np.int32:
if clampparams_data[
clampparamkey] > 0:
clampparams_data[
clampparamkey] = int(
1)
else:
clampparams_data[
clampparamkey] = int(
0)
else:
clampparams_data[
clampparamkey] = float(
clampparams_data[
clampparamkey])
clampparams_data_new[
clampparamkey.lower(
)] = clampparams_data[
clampparamkey]
#%
sweepmetadata_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'recording_mode':
recording_mode,
'sample_rate':
np.round(1 / np.median(
np.diff(
ephisdata['metadata']
[1]['values'])))
}
sweepmetadata_data.update(
clampparams_data_new)
sweepdata_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'response_trace':
ephisdata['data'][dataidx, :],
'response_units':
ephisdata['metadata'][0]
['cols'][dataidx]['units']
}
sweepstimulus_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'stimulus_trace':
ephisdata['data'][
commandidx, :],
'stimulus_units':
ephisdata['metadata'][0]
['cols'][commandidx]['units']
}
#print('waiting')
#timer.sleep(10000)
try:
ephys_patch.Sweep().insert1(
sweep_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try: # maybe it's a duplicate..
ephys_patch.ClampParams(
).insert1(
clampparams_data_new,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepMetadata(
).insert1(
sweepmetadata_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepResponse(
).insert1(
sweepdata_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepStimulus(
).insert1(
sweepstimulus_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
#%
if 'OrcaFlashExposure' in ephisdata.keys(
):
sweepimagingexposuredata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'imaging_exposure_trace':
ephisdata[
'OrcaFlashExposure']
}
try:
ephys_patch.SweepImagingExposure(
).insert1(
sweepimagingexposuredata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
if 'Temperature' in ephisdata.keys(
):
sweeptemperaturedata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'temperature_trace':
ephisdata['Temperature'] *
10,
'temperature_units':
'degC'
}
try:
ephys_patch.SweepTemperature(
).insert1(
sweeptemperaturedata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
if 'LED525' in ephisdata.keys():
sweepLEDdata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'imaging_led_trace':
ephisdata['LED525']
}
try:
ephys_patch.SweepLED(
).insert1(
sweepLEDdata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
age_first_surgery_list = list()
expression_time_list = list()
ml_list = list()
ap_list=list()
depth_list = list()
for subject_id in subject_ids:
virus_ids = (lab.Surgery.VirusInjection()&'subject_id = {}'.format(subject_id)).fetch('virus_id')
virus_id = virus_ids[0]
virus_name = (lab.Virus()&'virus_id = {}'.format(virus_id)).fetch1('virus_name')
if virus_name != 'syn-FLEX-Voltron-ST':
continue
if len(lab.Surgery()&'subject_id = {}'.format(subject_id))!=2:
print('only {} surgery for {}'.format(len(lab.Surgery()&'subject_id = {}'.format(subject_id)),subject_id))
continue
movie_num_list.append(len(imaging.Movie()&'subject_id = {}'.format(subject_id)))
date_of_birth,sex = (lab.Subject()&'subject_id = {}'.format(subject_id)).fetch1('date_of_birth','sex')
surgery_times = (lab.Surgery()&'subject_id = {}'.format(subject_id)).fetch('start_time')
ml,ap = (lab.Surgery.Procedure()&'subject_id = {}'.format(subject_id)).fetch1('ml_location','ap_location')
ml_list.append(int(ml))
ap_list.append(int(ap))
age_first_surgery_list.append((np.min(surgery_times).date()-date_of_birth).days)
expression_time_list.append(np.diff(surgery_times)[0].days)
sex_list.append(sex)
depth_list.extend((ephys_patch.Cell()&'subject_id = {}'.format(subject_id)).fetch('depth'))
depth_list = np.asarray(depth_list)
ap_list = np.asarray(ap_list)
ml_list = np.asarray(ml_list)
movie_num_list = np.asarray(movie_num_list)
sex_list = np.asarray(sex_list)
age_first_surgery_list = np.asarray(age_first_surgery_list)