from metadatastore.mds import MDSRO
from portable_mds.mongoquery.mds import MDS
source_config = {'host': 'localhost',
'port': 27017,
'database': 'metadatastore_production_v1',
'timezone': 'US/Eastern'}
dest_config = {'directory': 'some_directory',
'timezone': 'US/Eastern'}
source = MDSRO(source_config) # a read-only metadatastore object
dest = MDS(dest_config)
for run_start in source.find_run_starts():
dest.insert_run_start(**run_start)
for desc in source.find_descriptors(run_start=run_start):
events = source.get_events_generator(descriptor=desc)
dest.insert_descriptor(**desc)
dest.bulk_insert_events(desc, events)
dest.insert_run_stop(**source.stop_by_start(run_start))
from metadatastore.mds import MDSRO
from portable_mds.mongoquery.mds import MDS
source_config = {
'host': 'localhost',
'port': 27017,
'database': 'metadatastore_production_v1',
'timezone': 'US/Eastern'
}
dest_config = {'directory': 'some_directory', 'timezone': 'US/Eastern'}
source = MDSRO(source_config) # a read-only metadatastore object
dest = MDS(dest_config)
for run_start in source.find_run_starts():
dest.insert_run_start(**run_start)
for desc in source.find_descriptors(run_start=run_start):
events = source.get_events_generator(descriptor=desc)
dest.insert_descriptor(**desc)
dest.bulk_insert_events(desc, events)
dest.insert_run_stop(**source.stop_by_start(run_start))
descs = deque()
counts = deque()
total = old._descriptor_col.find().count()
old_descs = tqdm(old.find_descriptors(), unit='descriptors', total=total)
new_descs = new.find_descriptors()
for o, n in zip(old_descs, new_descs):
d_raw = next(old._descriptor_col.find({'uid': o['uid']}))
num_events = old._event_col.find({'descriptor_id': d_raw['_id']}).count()
assert o == n
descs.append(o)
counts.append(num_events)
total = sum(counts)
with tqdm(total=total, unit='events') as pbar:
for desc, num_events in zip(descs, counts):
old_events = old.get_events_generator(descriptor=desc,
convert_arrays=False)
new_events = new.get_events_generator(descriptor=desc,
convert_arrays=False)
try:
for ev in zip(old_events, new_events):
try:
assert o == n
except KeyError:
print(o)
print(n)
except KeyError:
print('descriptor uid', desc['uid'])
print('new events', new_events, list(new_events))
print('old events', old_events, list(old_events))
pbar.update(num_events)
print('All verified')
descs = deque()
counts = deque()
total = old._descriptor_col.find().count()
old_descs = tqdm(old.find_descriptors(), unit='descriptors', total=total)
new_descs = new.find_descriptors()
for o, n in zip(old_descs, new_descs):
d_raw = next(old._descriptor_col.find({'uid': o['uid']}))
num_events = old._event_col.find({'descriptor_id': d_raw['_id']}).count()
assert o == n
descs.append(o)
counts.append(num_events)
total = sum(counts)
with tqdm(total=total, unit='events') as pbar:
for desc, num_events in zip(descs, counts):
old_events = old.get_events_generator(descriptor=desc,
convert_arrays=False)
new_events = new.get_events_generator(descriptor=desc,
convert_arrays=False)
try:
for ev in zip(old_events, new_events):
try:
assert o == n
except KeyError:
print(o)
print(n)
except KeyError:
print('descriptor uid', desc['uid'])
print('new events', new_events, list(new_events))
print('old events', old_events, list(old_events))
pbar.update(num_events)
print('All verified')
