from metadatastore.mds import MDSRO from portable_mds.mongoquery.mds import MDS source_config = {'host': 'localhost', 'port': 27017, 'database': 'metadatastore_production_v1', 'timezone': 'US/Eastern'} dest_config = {'directory': 'some_directory', 'timezone': 'US/Eastern'} source = MDSRO(source_config) # a read-only metadatastore object dest = MDS(dest_config) for run_start in source.find_run_starts(): dest.insert_run_start(**run_start) for desc in source.find_descriptors(run_start=run_start): events = source.get_events_generator(descriptor=desc) dest.insert_descriptor(**desc) dest.bulk_insert_events(desc, events) dest.insert_run_stop(**source.stop_by_start(run_start))
from metadatastore.mds import MDSRO from portable_mds.mongoquery.mds import MDS source_config = { 'host': 'localhost', 'port': 27017, 'database': 'metadatastore_production_v1', 'timezone': 'US/Eastern' } dest_config = {'directory': 'some_directory', 'timezone': 'US/Eastern'} source = MDSRO(source_config) # a read-only metadatastore object dest = MDS(dest_config) for run_start in source.find_run_starts(): dest.insert_run_start(**run_start) for desc in source.find_descriptors(run_start=run_start): events = source.get_events_generator(descriptor=desc) dest.insert_descriptor(**desc) dest.bulk_insert_events(desc, events) dest.insert_run_stop(**source.stop_by_start(run_start))
total = old._runstart_col.find().count() old_starts = tqdm(old.find_run_starts(), unit='start docs', total=total, leave=True) new_starts = new.find_run_starts() for o, n in zip(old_starts, new_starts): compare(o, n) total = old._runstop_col.find().count() old_stops = tqdm(old.find_run_stops(), unit='stop docs', total=total) new_stops = new.find_run_stops() for o, n in zip(old_stops, new_stops): compare(o, n) descs = deque() counts = deque() total = old._descriptor_col.find().count() old_descs = tqdm(old.find_descriptors(), unit='descriptors', total=total) new_descs = new.find_descriptors() for o, n in zip(old_descs, new_descs): d_raw = next(old._descriptor_col.find({'uid': o['uid']})) num_events = old._event_col.find({'descriptor_id': d_raw['_id']}).count() assert o == n descs.append(o) counts.append(num_events) total = sum(counts) with tqdm(total=total, unit='events') as pbar: for desc, num_events in zip(descs, counts): old_events = old.get_events_generator(descriptor=desc, convert_arrays=False) new_events = new.get_events_generator(descriptor=desc, convert_arrays=False)
unit='start docs', total=total, leave=True) new_starts = new.find_run_starts() for o, n in zip(old_starts, new_starts): compare(o, n) total = old._runstop_col.find().count() old_stops = tqdm(old.find_run_stops(), unit='stop docs', total=total) new_stops = new.find_run_stops() for o, n in zip(old_stops, new_stops): compare(o, n) descs = deque() counts = deque() total = old._descriptor_col.find().count() old_descs = tqdm(old.find_descriptors(), unit='descriptors', total=total) new_descs = new.find_descriptors() for o, n in zip(old_descs, new_descs): d_raw = next(old._descriptor_col.find({'uid': o['uid']})) num_events = old._event_col.find({'descriptor_id': d_raw['_id']}).count() assert o == n descs.append(o) counts.append(num_events) total = sum(counts) with tqdm(total=total, unit='events') as pbar: for desc, num_events in zip(descs, counts): old_events = old.get_events_generator(descriptor=desc, convert_arrays=False) new_events = new.get_events_generator(descriptor=desc, convert_arrays=False)