def process_alyxraw_histology(filename='/data/alyxfull.json',
models=ALYX_HISTOLOGY_MODELS):
'''
Ingest all histology entries in a particular alyx dump, regardless of the current status.
'''
ingest_alyx_raw.insert_to_alyxraw(
ingest_alyx_raw.get_alyx_entries(filename=filename, models=models))
def process_updates(pks, current_dump='/data/alyxfull.json'):
'''
Update the all the fields in givens a set of pks
:param pks: uuids where an update is needed
:param current_dump: the latest
'''
print('Deleting from alyxraw...')
delete_update_entries.delete_entries_from_alyxraw(
modified_pks_important=pks)
print('Deleting from shadow membership...')
delete_update_entries.delete_entries_from_membership(pks)
print('Ingesting alyxraw...')
ingest_alyx_raw.insert_to_alyxraw(
ingest_alyx_raw.get_alyx_entries(current_dump, new_pks=pks))
print('Ingesting into shadow tables...')
ingest_shadow.main(excluded_tables=['DataSet', 'FileRecord'])
print('Ingesting into shadow membership tables...')
ingest_membership.main(pks)
print('Ingesting alyx real...')
ingest_real.main(excluded_tables=['DataSet', 'FileRecord'])
print('Updating field...')
delete_update_entries.update_entries_from_real_tables(pks)
def process_alyxraw_qc(
filename='/data/alyxfull.json',
models=['actions.session', 'experiments.probeinsertion']):
'''
Ingest all qc entries in a particular alyx dump, regardless of the current status.
'''
ingest_alyx_raw.insert_to_alyxraw(ingest_alyx_raw.get_alyx_entries(
filename=filename, models=models),
alyx_type='part')
def process_public():
from ibl_pipeline import public
from ibl_pipeline.common import subject, acquisition
ingest_alyx_raw.insert_to_alyxraw(ingest_alyx_raw.get_alyx_entries())
excluded_tables = [
'Weighing', 'WaterType', 'WaterAdministration', 'WaterRestriction',
'ProbeModel', 'ProbeInsertion', 'ProbeTrajectory'
]
ingest_shadow.main(excluded_tables=excluded_tables)
excluded_membership_tables = [
'WaterRestrictionUser',
'WaterRestrictionProcedure',
'SurgeryUser',
'WaterAdministrationSession',
]
ingest_membership.main(excluded_tables=excluded_membership_tables)
ingest_real.main(excluded_tables=excluded_tables +
excluded_membership_tables,
public=True)
# delete non-releasing tables
from ibl_pipeline.ingest import InsertBuffer
table = InsertBuffer(acquisition.Session)
for key in tqdm((acquisition.Session - public.PublicSession -
behavior.TrialSet).fetch('KEY')):
table.delete1(key)
if table.flush_delete(chunksz=100):
print('Deleted 100 sessions')
table.flush_delete()
print('Deleted the rest of the sessions')
subjs = subject.Subject & acquisition.Session
for key in tqdm((subject.Subject - public.PublicSubjectUuid -
subjs.proj()).fetch('KEY')):
(subject.Subject & key).delete()
excluded_behavior_tables = [
'AmbientSensorData', 'Settings', 'SessionDelay'
]
populate_behavior.main(excluded_tables=excluded_behavior_tables)
def insert_to_update_alyxraw(filename=None, delete_tables=False, models=None):
dj.config['safemode'] = False
if not models:
raise ValueError('Argument models is required, \
str of an alyx model or a list of alyx models')
if delete_tables:
print('Deleting alyxraw update...')
alyxraw_update.AlyxRaw.Field.delete_quick()
alyxraw_update.AlyxRaw.delete_quick()
ingest_alyx_raw.insert_to_alyxraw(ingest_alyx_raw.get_alyx_entries(
filename=filename, models=models),
alyxraw_module=alyxraw_update)
def main(populate_only=False):
if not populate_only:
logger.log(25, 'Ingesting alyxraw...')
ingest_alyx_raw.insert_to_alyxraw(ingest_alyx_raw.get_alyx_entries())
logger.log(25, 'Ingesting shadow tables...')
ingest_shadow.main()
logger.log(25, 'Ingesting shadow membership...')
ingest_membership.main()
logger.log(25, 'Copying to real tables...')
ingest_real.main()
logger.log(25, 'Deleting the non published records...')
delete_non_published_records()
logger.log(25, 'Processing behavior...')
populate_behavior.main(backtrack_days=1000)
logger.log(25, 'Processing wheel...')
populate_wheel.main(backtrack_days=1000)
def main(populate_only=False, populate_wheel=False, populate_ephys_histology=False):
"""This function process the all the steps to get data ingested into the public database, needs rewriting
to load data from sql dump instead.
Args:
populate_only (bool, optional): If True, only populate table; if False, start from the beginning
and load entries from alyx dump in the folder /data. Defaults to False.
populate_wheel (bool, optional): If True, populate wheel
populate_ephys_histology (bool, optional): If True, populate ephys and histology tables
"""
if not populate_only:
logger.log(25, 'Ingesting alyxraw...')
ingest_alyx_raw.insert_to_alyxraw(
ingest_alyx_raw.get_alyx_entries())
logger.log(25, 'Ingesting shadow tables...')
ingest_shadow.main()
logger.log(25, 'Ingesting shadow membership...')
ingest_membership.main()
logger.log(25, 'Copying to real tables...')
ingest_real.main()
logger.log(25, 'Deleting the non published records...')
delete_non_published_records()
logger.log(25, 'Processing behavior...')
populate_behavior.main(backtrack_days=1000)
if populate_wheel:
logger.log(25, 'Processing wheel...')
populate_wheel.main(backtrack_days=1000)
if populate_ephys_histology:
logger.log(25, 'Processing ephys...')
populate_ephys.main()
logger.log(25, 'Processing histology...')
process_histology.populate_real_tables()
import datajoint as dj
import json
import os.path as path
import sys
from ibl_pipeline.ingest import QueryBuffer
from ibl_pipeline.ingest import reference as reference_ingest
from ibl_pipeline import reference
from ibl_pipeline.process.ingest_alyx_raw import get_alyx_entries
from tqdm import tqdm
import pandas as pd
import numpy as np
keys = get_alyx_entries(models='experiments.brainregion')
atlas = pd.read_csv('/data/allen_structure_tree.csv')
def ingest_all():
ib_brainregion = QueryBuffer(reference_ingest.BrainRegion)
for key in tqdm(keys, position=0):
fields = key['fields']
graph_order = atlas[atlas['id'] == key['pk']]['graph_order']
if np.isnan(graph_order.to_list()[0]):
graph_order = None
else:
graph_order = int(graph_order)
ib_brainregion.add_to_queue1(
dict(brain_region_pk=key['pk'],
def process_new(previous_dump=None,
latest_dump=None,
job_date=datetime.date.today().strftime('%Y-%m-%d'),
timezone='other'):
job_key = dict(
job_date=job_date,
job_timezone=timezone,
)
if previous_dump is None:
previous_dump = path.join('/', 'data', 'alyxfull.json.last')
if latest_dump is None:
latest_dump = path.join('/', 'data', 'alyxfull.json')
if not (job.Job & job_key):
print('Comparing json dumps ...')
create_ingest_task.compare_json_dumps(previous_dump, latest_dump)
created_pks, modified_pks, deleted_pks, modified_pks_important = (
job.Job & job_key).fetch1('created_pks', 'modified_pks', 'deleted_pks',
'modified_pks_important')
print('Deleting modified entries from alyxraw and shadow tables...')
start = datetime.datetime.now()
delete_update_entries.delete_entries_from_alyxraw(modified_pks,
modified_pks_important)
ingest_status(job_key,
'Delete alyxraw',
start,
end=datetime.datetime.now())
print('Deleting modified entries from membership tables...')
start = datetime.datetime.now()
delete_update_entries.delete_entries_from_membership(
modified_pks_important)
ingest_status(job_key,
'Delete shadow membership',
start,
end=datetime.datetime.now())
print('Ingesting into alyxraw...')
start = datetime.datetime.now()
ingest_alyx_raw.insert_to_alyxraw(
ingest_alyx_raw.get_alyx_entries(latest_dump,
new_pks=created_pks + modified_pks))
ingest_status(job_key,
'Ingest alyxraw',
start,
end=datetime.datetime.now())
print('Ingesting into shadow tables...')
start = datetime.datetime.now()
ingest_shadow.main(modified_pks=modified_pks_important)
ingest_status(job_key, 'Ingest shadow', start, end=datetime.datetime.now())
print('Ingesting into shadow membership tables...')
start = datetime.datetime.now()
ingest_membership.main(created_pks + modified_pks_important)
ingest_status(job_key,
'Ingest shadow membership',
start,
end=datetime.datetime.now())
print('Ingesting alyx real...')
start = datetime.datetime.now()
ingest_real.main()
ingest_status(job_key, 'Ingest real', start, end=datetime.datetime.now())
print('Updating fields...')
start = datetime.datetime.now()
delete_update_entries.update_entries_from_real_tables(
modified_pks_important)
ingest_status(job_key, 'Update fields', start, end=datetime.datetime.now())
print('Ingesting behavior...')
start = datetime.datetime.now()
populate_behavior.main(backtrack_days=30)
ingest_status(job_key,
'Populate behavior',
start,
end=datetime.datetime.now())
