from metadatastore.mds import MDSRO
from portable_mds.mongoquery.mds import MDS
source_config = {'host': 'localhost',
'port': 27017,
'database': 'metadatastore_production_v1',
'timezone': 'US/Eastern'}
dest_config = {'directory': 'some_directory',
'timezone': 'US/Eastern'}
source = MDSRO(source_config) # a read-only metadatastore object
dest = MDS(dest_config)
for run_start in source.find_run_starts():
dest.insert_run_start(**run_start)
for desc in source.find_descriptors(run_start=run_start):
events = source.get_events_generator(descriptor=desc)
dest.insert_descriptor(**desc)
dest.bulk_insert_events(desc, events)
dest.insert_run_stop(**source.stop_by_start(run_start))
from metadatastore.mds import MDSRO
from portable_mds.mongoquery.mds import MDS
source_config = {
'host': 'localhost',
'port': 27017,
'database': 'metadatastore_production_v1',
'timezone': 'US/Eastern'
}
dest_config = {'directory': 'some_directory', 'timezone': 'US/Eastern'}
source = MDSRO(source_config) # a read-only metadatastore object
dest = MDS(dest_config)
for run_start in source.find_run_starts():
dest.insert_run_start(**run_start)
for desc in source.find_descriptors(run_start=run_start):
events = source.get_events_generator(descriptor=desc)
dest.insert_descriptor(**desc)
dest.bulk_insert_events(desc, events)
dest.insert_run_stop(**source.stop_by_start(run_start))
total = old._runstart_col.find().count()
old_starts = tqdm(old.find_run_starts(), unit='start docs', total=total,
leave=True)
new_starts = new.find_run_starts()
for o, n in zip(old_starts, new_starts):
compare(o, n)
total = old._runstop_col.find().count()
old_stops = tqdm(old.find_run_stops(), unit='stop docs', total=total)
new_stops = new.find_run_stops()
for o, n in zip(old_stops, new_stops):
compare(o, n)
descs = deque()
counts = deque()
total = old._descriptor_col.find().count()
old_descs = tqdm(old.find_descriptors(), unit='descriptors', total=total)
new_descs = new.find_descriptors()
for o, n in zip(old_descs, new_descs):
d_raw = next(old._descriptor_col.find({'uid': o['uid']}))
num_events = old._event_col.find({'descriptor_id': d_raw['_id']}).count()
assert o == n
descs.append(o)
counts.append(num_events)
total = sum(counts)
with tqdm(total=total, unit='events') as pbar:
for desc, num_events in zip(descs, counts):
old_events = old.get_events_generator(descriptor=desc,
convert_arrays=False)
new_events = new.get_events_generator(descriptor=desc,
convert_arrays=False)
unit='start docs',
total=total,
leave=True)
new_starts = new.find_run_starts()
for o, n in zip(old_starts, new_starts):
compare(o, n)
total = old._runstop_col.find().count()
old_stops = tqdm(old.find_run_stops(), unit='stop docs', total=total)
new_stops = new.find_run_stops()
for o, n in zip(old_stops, new_stops):
compare(o, n)
descs = deque()
counts = deque()
total = old._descriptor_col.find().count()
old_descs = tqdm(old.find_descriptors(), unit='descriptors', total=total)
new_descs = new.find_descriptors()
for o, n in zip(old_descs, new_descs):
d_raw = next(old._descriptor_col.find({'uid': o['uid']}))
num_events = old._event_col.find({'descriptor_id': d_raw['_id']}).count()
assert o == n
descs.append(o)
counts.append(num_events)
total = sum(counts)
with tqdm(total=total, unit='events') as pbar:
for desc, num_events in zip(descs, counts):
old_events = old.get_events_generator(descriptor=desc,
convert_arrays=False)
new_events = new.get_events_generator(descriptor=desc,
convert_arrays=False)
