def get_created_deleted_pks(data0, data1):
old_pks = {_['pk'] for _ in data0}
new_pks = {_['pk'] for _ in data1}
return [pk for pk in sorted(new_pks - old_pks) if is_valid_uuid(pk)], \
[pk for pk in sorted(old_pks - new_pks) if is_valid_uuid(pk)]
def get_created_deleted_pks(data0, data1):
old_pks = {
_['pk']
for _ in data0
if not isinstance(_['pk'], int) and _['model'] not in EXCLUDED_MODELS
}
new_pks = {
_['pk']
for _ in data1
if not isinstance(_['pk'], int) and _['model'] not in EXCLUDED_MODELS
}
return [pk for pk in sorted(new_pks - old_pks) if is_valid_uuid(pk)], \
[pk for pk in sorted(old_pks - new_pks) if is_valid_uuid(pk)]
def get_modified_pks(data0, data1):
d0 = {_['pk']: json.dumps(_['fields'], sort_keys=True) for _ in data0}
d1 = {_['pk']: json.dumps(_['fields'], sort_keys=True) for _ in data1}
d0 = {k: v for k, v in d0.items() if k in d1.keys()}
d1 = {k: v for k, v in d1.items() if k in d0.keys()}
return [k for k in d0.keys() if d0[k] != d1[k] and is_valid_uuid(k)]
def delete_entries_from_alyxraw(pks_to_be_deleted=[],
modified_pks_important=[]):
'''
Delete entries from alyxraw and shadow membership_tables, excluding the membership table.
'''
print('Deleting alyxraw entries corresponding to file records...')
if pks_to_be_deleted:
if len(pks_to_be_deleted) > 5000:
file_record_fields = alyxraw.AlyxRaw.Field & \
'fname = "exists"' & 'fvalue = "false"'
else:
file_record_fields = alyxraw.AlyxRaw.Field & \
'fname = "exists"' & 'fvalue = "false"' & \
[{'uuid': pk} for pk in pks_to_be_deleted]
for key in tqdm(file_record_fields):
(alyxraw.AlyxRaw.Field & key).delete_quick()
if modified_pks_important:
pk_list = [{
'uuid': pk
} for pk in modified_pks_important if is_valid_uuid(pk)]
(alyxraw.AlyxRaw & 'model != "actions.session"' & pk_list).delete()
(alyxraw.AlyxRaw.Field & pk_list & 'fname!="start_time"' &
(alyxraw.AlyxRaw & 'model="actions.session"')).delete_quick()
def delete_entries_from_membership(pks_to_be_deleted):
'''
Delete entries from shadow membership membership_tables
'''
for t in membership_tables:
ingest_mod = t['dj_parent_table'].__module__
table_name = t['dj_parent_table'].__name__
mem_table_name = t['dj_current_table'].__name__
print(f'Deleting from table {mem_table_name} ...')
real_table = eval(ingest_mod.replace('ibl_pipeline.ingest.', '') + '.' + table_name)
(t['dj_current_table'] &
(real_table &
[{t['dj_parent_uuid_name']:pk}
for pk in pks_to_be_deleted if is_valid_uuid(pk)]).fetch('KEY')).delete()
def delete_entries_from_alyxraw(pks_to_be_deleted=[], modified_pks_important=[]):
'''
Delete entries from alyxraw and shadow membership_tables, excluding the membership table.
'''
print('Deleting alyxraw entries corresponding to file records...')
if pks_to_be_deleted:
if len(pks_to_be_deleted) > 5000:
file_record_fields = alyxraw.AlyxRaw.Field & \
'fname = "exists"' & 'fvalue = "false"'
else:
file_record_fields = alyxraw.AlyxRaw.Field & \
'fname = "exists"' & 'fvalue = "false"' & \
[{'uuid': pk} for pk in pks_to_be_deleted]
for key in tqdm(file_record_fields):
(alyxraw.AlyxRaw.Field & key).delete_quick()
if modified_pks_important:
pk_list = [{'uuid': pk} for pk in modified_pks_important
if is_valid_uuid(pk)]
if len(pk_list) > 1000:
print('Long pk list, deleting from alyxraw.AlyxRaw ...')
alyxraw_buffer = QueryBuffer(alyxraw.AlyxRaw & 'model != "actions.session"')
for pk in tqdm(pk_list):
alyxraw_buffer.add_to_queue1(pk)
alyxraw_buffer.flush_delete(chunksz=50, quick=False)
alyxraw_buffer.flush_delete(quick=False)
# Delete session fields without deleting the AlyxRaw entries and start time field.
# This is to handle the case where uuid is not changed but start time changed for 1 sec.
print('Long pk list, deleting from alyxraw.AlyxRaw.Field ...')
alyxraw_field_buffer = QueryBuffer(
alyxraw.AlyxRaw.Field & 'fname!="start_time"' &
(alyxraw.AlyxRaw & 'model="actions.session"'))
for pk in tqdm(pk_list):
alyxraw_field_buffer.add_to_queue1(pk)
alyxraw_field_buffer.flush_delete(chunksz=50, quick=True)
alyxraw_field_buffer.flush_delete(quick=True)
def get_important_pks(pks, return_original_dict=False):
'''
Filter out modified keys that belongs to data.filerecord and jobs.task
:params modified_keys: list of pks
:params optional return original_dict: boolean, if True, return the list of dictionaries with uuids to be the key
:returns pks_important: list of filtered pks
:returns pks_dict: list of dictionary with uuid as the key
'''
pks = [pk for pk in pks if is_valid_uuid(pk)]
pks_dict = [{'uuid': pk} for pk in pks]
pks_unimportant = [
str(pk['uuid'])
for pk in (alyxraw.AlyxRaw & 'model in ("data.filerecord", "jobs.task")' & pks_dict).fetch('KEY')]
pks_important = list(set(pks) - set(pks_unimportant))
if return_original_dict:
return pks_important, pks_dict
else:
return pks_important
def get_important_pks(pks, return_original_dict=False):
'''
Filter out modified keys that belongs to data.filerecord and jobs.task
:params modified_keys: list of pks
:params optional return original_dict: boolean, if True, return the list of dictionaries with uuids to be the key
:returns pks_important: list of filtered pks
:returns pks_dict: list of dictionary with uuid as the key
'''
pks = [pk for pk in pks if is_valid_uuid(pk)]
pks_dict = [{'uuid': pk} for pk in pks]
models_ignored = '"data.dataset", "data.filerecord", "jobs.task", "actions.wateradministration", "experiments.trajectoryestimate", "experiments.channel"'
if len(pks) < 1000:
pks_unimportant = [
str(pk['uuid'])
for pk in (alyxraw.AlyxRaw &
f'model in ({models_ignored})' &
pks_dict).fetch('KEY')]
else:
buffer = QueryBuffer(
alyxraw.AlyxRaw & f'model in ({models_ignored})')
for pk in tqdm(pks_dict):
buffer.add_to_queue1(pk)
buffer.flush_fetch('KEY', chunksz=200)
buffer.flush_fetch('KEY')
pks_unimportant = [str(pk['uuid']) for pk in buffer.fetched_results]
pks_important = list(set(pks) - set(pks_unimportant))
if return_original_dict:
return pks_important, pks_dict
else:
return pks_important
def ingest_membership_table(dj_current_table,
alyx_parent_model,
alyx_field,
dj_parent_table,
dj_other_table,
dj_parent_fields,
dj_other_field,
dj_parent_uuid_name,
dj_other_uuid_name,
renamed_other_field_name=None,
new_pks=None):
'''
Ingest shadow membership table.
This function works for the pattern that an alyx parent model contain one or multiple entries of one field
that have the information in the membership table.
Arguments: dj_current_table : datajoint table object, current membership table to ingest
alyx_parent_model: string, model name inside alyx that contains information of the current table.
alyx_field : field of alyx that contains information of current table
dj_parent_table : datajoint parent table, corresponding to alyx parent model
dj_other_table : datajoint other table to fetch the field from
dj_parent_fields : string or list of strings, field names to be fetched from the parent table
dj_other_field : string, the field table to be fetched from the other table
dj_parent_uuid_name: string, uuid id name of the parent table
dj_other_uuid_name: string, uuid id name of the other table
renamed_other_field_name: string the other field name sometimes renamed in the real table,
the default is None if the field is not renamed
new_pks : list of strings of valid uuids, this is the new entries to process, the
default is None if all entries are inserted.
'''
if new_pks:
restr = [{'uuid': pk} for pk in new_pks if is_valid_uuid(pk)]
else:
restr = {}
alyxraw_to_insert = (alyxraw.AlyxRaw & restr & {
'model': alyx_parent_model
}).fetch('KEY')
if not alyxraw_to_insert:
return
alyx_field_entries = alyxraw.AlyxRaw.Field & alyxraw_to_insert & \
{'fname': alyx_field} & 'fvalue!="None"'
keys = (alyxraw.AlyxRaw
& alyx_field_entries).proj(**{dj_parent_uuid_name: 'uuid'})
if type(dj_parent_fields) == str:
dj_parent_fields = [dj_parent_fields]
for key in keys:
if not dj_parent_table & key:
print(
f'The entry {key} is not parent table {dj_parent_table.__name__}'
)
continue
entry_base = (dj_parent_table & key).fetch(*dj_parent_fields,
as_dict=True)[0]
key['uuid'] = key[dj_parent_uuid_name]
uuids = grf(key,
alyx_field,
multiple_entries=True,
model=alyx_parent_model)
if len(uuids):
for uuid in uuids:
if uuid == 'None':
continue
else:
if not dj_other_table & {dj_other_uuid_name: uuid}:
print(
f'The uuid {uuid} is not datajoint table {dj_other_table.__name__}'
)
continue
entry = entry_base.copy()
field_value = (dj_other_table & {
dj_other_uuid_name: uuid
}).fetch1(dj_other_field)
if renamed_other_field_name:
entry[renamed_other_field_name] = field_value
else:
entry[dj_other_field] = field_value
dj_current_table.insert1(entry, skip_duplicates=True)
