def build_pymongo_backed_broker():
"""Provide a function level scoped MDS instance talking to
temporary database on localhost:27017 with v1 schema.
"""
from databroker.broker import Broker
from metadatastore.mds import MDS
from filestore.utils import create_test_database
from filestore.fs import FileStore
from filestore.handlers import NpyHandler
db_name = "mds_testing_disposable_{}".format(str(uuid.uuid4()))
mds_test_conf = dict(database=db_name, host='localhost',
port=27017, timezone='US/Eastern')
try:
# nasty details: to save MacOS user
mds = MDS(mds_test_conf, 1, auth=False)
except TypeError:
mds = MDS(mds_test_conf, 1)
db_name = "fs_testing_base_disposable_{}".format(str(uuid.uuid4()))
fs_test_conf = create_test_database(host='localhost',
port=27017,
version=1,
db_template=db_name)
fs = FileStore(fs_test_conf, version=1)
fs.register_handler('npy', NpyHandler)
db = Broker(mds, fs)
insert_imgs(db.mds, db.fs, 1, (2048,2048))
return db
def test_pickle():
from metadatastore.mds import MDS
md = MDS(config={'host': 'portland'}, version=1)
md2 = pickle.loads(pickle.dumps(md))
assert md.version == md2.version
assert md.config == md2.config
def mds(request):
db_template = "mds_testing_disposable_{}".format(str(uuid.uuid4()))
test_conf = create_test_database('localhost', db_template=db_template)
mds = MDS(test_conf)
def delete_dm():
print("DROPPING DB")
mds._connection.drop_database(test_conf['database'])
request.addfinalizer(delete_dm)
return mds
def mds_all(request):
'''Provide a function level scoped FileStore instance talking to
temporary database on localhost:27017 with both v0 and v1.
'''
db_name = "mds_testing_disposable_{}".format(str(uuid.uuid4()))
test_conf = dict(database=db_name,
host='localhost',
port=27017,
timezone='US/Eastern',
mongo_user='******',
mongo_pwd='jerry')
ver = request.param
mds = MDS(test_conf, ver, auth=AUTH)
def delete_dm():
print("DROPPING DB")
mds._connection.drop_database(db_name)
request.addfinalizer(delete_dm)
return mds
# Set up the logbook. This configures bluesky's summaries of
# data acquisition (scan type, ID, etc.).
# Make ophyd listen to pyepics.
from ophyd import setup_ophyd
setup_ophyd()
from metadatastore.mds import MDS
# from metadataclient.mds import MDS
from databroker import Broker
from databroker.core import register_builtin_handlers
from filestore.fs import FileStore
# pull from /etc/metadatastore/connection.yaml or
# /home/BLUSER/.config/metdatastore/connection.yml
mds = MDS({'host': 'xf21id1-ca1',
'database': 'metadatastore',
'port': 27017,
'timezone': 'US/Eastern'}, auth=False)
# mds = MDS({'host': CA, 'port': 7770})
# pull configuration from /etc/filestore/connection.yaml or
# /home/BLUSER/.config/filestore/connection.yml
db = Broker(mds, FileStore({'host': 'xf21id1-ca1',
'port': 27017,
'database': 'filestore'}))
register_builtin_handlers(db.fs)
# Subscribe metadatastore to documents.
# If this is removed, data is not saved to metadatastore.
from bluesky.global_state import gs
gs.RE.subscribe('all', mds.insert)
pass
else:
print(o)
print(n)
raise
old_config = dict(database=OLD_DATABASE,
host='localhost',
port=27017,
timezone='US/Eastern')
new_config = old_config.copy()
new_config['database'] = NEW_DATABASE
old = MDSRO(version=0, config=old_config)
new = MDS(version=1, config=new_config)
total = old._runstart_col.find().count()
old_starts = tqdm(old.find_run_starts(), unit='start docs', total=total,
leave=True)
new_starts = new.find_run_starts()
for o, n in zip(old_starts, new_starts):
compare(o, n)
total = old._runstop_col.find().count()
old_stops = tqdm(old.find_run_stops(), unit='stop docs', total=total)
new_stops = new.find_run_stops()
for o, n in zip(old_stops, new_stops):
compare(o, n)
descs = deque()
counts = deque()
#from hxntools.handlers import register
#import filestore
from metadatastore.mds import MDS
from databroker import Broker
from filestore.fs import FileStore
# database #1
_mds_config = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'datastore-new',
'timezone': 'US/Eastern'}
mds = MDS(_mds_config)
_fs_config = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'filestore-new'}
db1 = Broker(mds, FileStore(_fs_config))
# database #2
_mds_config = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'datastore-1',
'timezone': 'US/Eastern'}
mds = MDS(_mds_config)
_fs_config = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'filestore-1'}
db2 = Broker(mds, FileStore(_fs_config))
# database old
_mds_config_old = {'host': 'xf03id-ca1',
'port': 27017,
# Make ophyd listen to pyepics.
from ophyd import setup_ophyd
setup_ophyd()
# Subscribe metadatastore to documents.
# If this is removed, data is not saved to metadatastore.
from metadatastore.mds import MDS
from filestore.fs import FileStore
from databroker import Broker
mds = MDS({'host': 'xf17bm-ioc1',
'port': 27017,
'database': 'metadatastore-production-v1',
'timezone': 'US/Eastern'})
fs = FileStore({'host': 'xf17bm-ioc1',
'port': 27017,
'database': 'filestore-production-v1'})
db = Broker(mds, fs)
# usage: db[-1] # i.e., one scan ago
from bluesky.global_state import gs
gs.RE.subscribe_lossless('all', mds.insert)
# At the end of every run, verify that files were saved and
# print a confirmation message.
# from bluesky.callbacks.broker import verify_files_saved
# gs.RE.subscribe('stop', post_run(verify_files_saved))
# Import matplotlib and put it in interactive mode.
import matplotlib.pyplot as plt
plt.ion()
os.environ['FS_HOST'] = 'localhost'
os.environ['FS_PORT'] = '27017'
os.environ['FS_DATABASE'] = 'filestore-production-v1'
# Connect to metadatastore and filestore.
from metadatastore.mds import MDS, MDSRO
from filestore.fs import FileStoreRO
from databroker import Broker
mds_config = {'host': 'localhost',
'port': 27017,
'database': 'metadatastore-production-v1',
'timezone': 'US/Eastern'}
fs_config = {'host': 'localhost',
'port': 27017,
'database': 'filestore-production-v1'}
mds = MDS(mds_config)
#mds_readonly = MDSRO(mds_config)
# As we aren't writing any files at the moment, use
# the readonly version
fs_readonly = FileStoreRO(fs_config)
db = Broker(mds, fs_readonly)
# Subscribe metadatastore to documents.
# If this is removed, data is not saved to metadatastore.
from bluesky.global_state import gs
gs.RE.subscribe('all', mds.insert)
# Import matplotlib and put it in interactive mode.
import matplotlib.pyplot as plt
from metadatastore.mds import MDS
from databroker import Broker
from databroker.core import register_builtin_handlers
from filestore.fs import FileStore
_mds_config = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'datastore-new',
'timezone': 'US/Eastern'}
mds = MDS(_mds_config, auth=False)
_fs_config = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'filestore-new'}
db_new = Broker(mds, FileStore(_fs_config))
_mds_config_old = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'datastore',
'timezone': 'US/Eastern'}
mds_old = MDS(_mds_config_old, auth=False)
_fs_config_old = {'host': 'xf03id-ca1',
'port': 27017,
'database': 'filestore'}
db_old = Broker(mds_old, FileStore(_fs_config_old))
from hxntools.handlers.xspress3 import Xspress3HDF5Handler
from hxntools.handlers.timepix import TimepixHDF5Handler
def main(target, source):
print('Database to be migrated {}'.format(source))
print('Database migrated to {}'.format(target))
target = target
rc = ipp.Client()
dview = rc[:]
with dview.sync_imports():
from metadatastore.mds import MDS, MDSRO
from collections import deque
old_config = {
'database': source,
'host': 'localhost',
'port': 27017,
'timezone': 'US/Eastern'}
new_config = {
'database': target,
'host': 'localhost',
'port': 27017,
'timezone': 'US/Eastern'}
old_t = MDSRO(version=0, config=old_config)
new_t = MDS(version=1, config=new_config)
def condition_config():
import time
global new, old
for md in [new, old]:
md._runstart_col.find_one()
md._runstop_col.find_one()
md._event_col.find_one()
md._descriptor_col.find_one()
time.sleep(1)
def invasive_checks():
global old, new
return (old._MDSRO__conn is None,
new._MDSRO__conn is None)
dview.push({'old': old_t, 'new': new_t})
dview.apply(condition_config)
print(list(dview.apply(invasive_checks)))
new_t._connection.drop_database(target)
# Drop all indexes on event collection to speed up insert.
# They will be rebuilt the next time an MDS(RO) object connects.
new_t._event_col.drop_indexes()
new = new_t
old = old_t
# old._runstart_col.drop_indexes()
total = old._runstart_col.find().count()
for start in tqdm(old.find_run_starts(), desc='start docs', total=total):
new.insert('start', start)
total = old._runstop_col.find().count()
for stop in tqdm(old.find_run_stops(), desc='stop docs', total=total):
try:
new.insert('stop', stop)
except RuntimeError:
print("error inserting run stop with uid {!r}".format(stop['uid']))
descs = deque()
counts = deque()
old._descriptor_col.drop_indexes()
total = old._descriptor_col.find().count()
for desc in tqdm(old.find_descriptors(), unit='descriptors', total=total):
d_raw = next(old._descriptor_col.find({'uid': desc['uid']}))
num_events = old._event_col.find(
{'descriptor_id': d_raw['_id']}).count()
new.insert('descriptor', desc)
out = dict(desc)
out['run_start'] = out['run_start']['uid']
descs.append(dict(desc))
counts.append(num_events)
new.clear_process_cache()
old.clear_process_cache()
def migrate_event_stream(desc_in, num_events):
import pymongo.errors
import time
global new, old
if num_events:
flag = True
# skip empty event stream of bulk insert raises
while flag:
flag = False
try:
events = old.get_events_generator(descriptor=desc_in,
convert_arrays=False)
events = iter(events)
l_cache = deque()
while True:
try:
for j in range(5000):
l_cache.append(next(events))
except StopIteration:
break
finally:
if l_cache:
new.bulk_insert_events(descriptor=desc_in,
events=l_cache)
l_cache.clear()
except KeyError:
print("here here, key error")
except pymongo.errors.AutoReconnect:
flag = True
time.sleep(10)
new.clear_process_cache()
old.clear_process_cache()
return num_events
v = rc.load_balanced_view()
amr = v.map(migrate_event_stream, descs, list(counts), ordered=False)
total = sum(counts)
with tqdm(total=total, unit='events') as pbar:
for res in amr:
pbar.update(res)
else:
print(o)
print(n)
raise
old_config = dict(database=OLD_DATABASE,
host='localhost',
port=27017,
timezone='US/Eastern')
new_config = old_config.copy()
new_config['database'] = NEW_DATABASE
old = MDSRO(version=0, config=old_config)
new = MDS(version=1, config=new_config)
total = old._runstart_col.find().count()
old_starts = tqdm(old.find_run_starts(),
unit='start docs',
total=total,
leave=True)
new_starts = new.find_run_starts()
for o, n in zip(old_starts, new_starts):
compare(o, n)
total = old._runstop_col.find().count()
old_stops = tqdm(old.find_run_stops(), unit='stop docs', total=total)
new_stops = new.find_run_stops()
for o, n in zip(old_stops, new_stops):
compare(o, n)
global proposal_id
global run_id
if username is None or proposal_id is None or run_id is None:
login()
return super().__call__(*args, **kwargs)
RE = CustomRunEngine()
gs.RE = RE
mds = MDS(
{
'host': 'xf16idc-ca',
'database': 'metadatastore_production_v1',
'port': 27017,
'timezone': 'US/Eastern'
},
auth=False)
db = Broker(
mds,
FileStore({
'host': 'xf16idc-ca',
'database': 'filestore',
'port': 27017
}))
register_builtin_handlers(db.fs)
RE.subscribe('all', mds.insert)