def test_export(db_factory, example_data):
documents = example_data()
metadatastore_db = db_factory()
asset_registry_db = db_factory()
serializer = Serializer(metadatastore_db, asset_registry_db)
for item in documents:
serializer(*item)
def __init__(self, **kwargs):
super(QRunEngine, self).__init__()
self.RE = RunEngine(context_managers=[],
during_task=DuringTask(),
**kwargs)
self.RE.subscribe(self.sigDocumentYield.emit)
# TODO: pull from settings plugin
from suitcase.mongo_normalized import Serializer
#TODO create single databroker db
#python-dotenv stores name-value pairs in .env (add to .gitginore)
username = os.getenv("USER_MONGO")
pw = os.getenv("PASSWD_MONGO")
try:
self.RE.subscribe(
Serializer(
f"mongodb://{username}:{pw}@localhost:27017/mds?authsource=mds",
f"mongodb://{username}:{pw}@localhost:27017/fs?authsource=fs"
))
except OperationFailure as err:
msg.notifyMessage("Could not connect to local mongo database.",
title="xicam.Acquire Error",
level=msg.ERROR)
msg.logError(err)
self.queue = PriorityQueue()
self.process_queue()
def test_update(db_factory, example_data):
documents = example_data()
metadatastore_db = db_factory()
asset_registry_db = db_factory()
serializer = Serializer(metadatastore_db, asset_registry_db)
for item in documents:
serializer(*item)
original = documents[0][1]
start = copy.deepcopy(original)
start['user'] = '******'
serializer.update('start', start)
real = metadatastore_db.get_collection('run_start').find_one({'uid': start['uid']})
real.pop('_id')
assert sanitize_doc(real) == sanitize_doc(start)
revision = metadatastore_db.get_collection('run_start_revisions').find_one({'document.uid': start['uid']})
assert revision['revision'] == 0
revision.pop('revision')
revision.pop('_id')
assert sanitize_doc(revision['document']) == sanitize_doc(original)
revision1 = copy.deepcopy(start)
start['user'] = '******'
serializer.update('start', start)
real = metadatastore_db.get_collection('run_start').find_one({'uid': start['uid']})
real.pop('_id')
assert sanitize_doc(real) == sanitize_doc(start)
revision = metadatastore_db.get_collection('run_start_revisions').find_one({'document.uid': start['uid'],
'revision': 1})
assert revision['revision'] == 1
revision.pop('revision')
revision.pop('_id')
assert sanitize_doc(revision['document']) == sanitize_doc(revision1)
def get_catalog():
RE = RunEngine()
mds = f"mongodb://localhost:27017/databroker-test-{uuid.uuid4()}"
fs = f"mongodb://localhost:27017/databroker-test-{uuid.uuid4()}"
serializer = Serializer(mds, fs)
for i in range(1, 5):
RE(scan([det], motor, -1, 1, 5 * i), serializer)
RE(count([random_img], 3), serializer)
catalog = BlueskyMongoCatalog(mds, fs)
return catalog
def test_duplicates(db_factory, example_data):
# Duplicate should not cause exceptions, and should be deduped.
documents = example_data()
metadatastore_db = db_factory()
asset_registry_db = db_factory()
serializer = Serializer(metadatastore_db, asset_registry_db)
for item in documents:
serializer(*item)
for item in documents:
serializer(*item)
# Modify a document, check that inserting a document with uid,
# but different content raises.
documents[0][1]['new_key'] = 'new_value'
with pytest.raises(DuplicateKeyError):
for item in documents:
serializer(*item)
def test_index_creation(db_factory):
db = db_factory()
print(type(db))
metadatastore_db = db_factory()
asset_registry_db = db_factory()
Serializer(metadatastore_db, asset_registry_db)
indexes = asset_registry_db.resource.index_information()
assert len(indexes.keys()) == 3
assert indexes['uid_1']['unique']
assert indexes['resource_id_1']
indexes = asset_registry_db.datum.index_information()
assert len(indexes.keys()) == 3
assert indexes['datum_id_1']['unique']
assert indexes['resource_1']
indexes = metadatastore_db.run_start.index_information()
assert len(indexes.keys()) == 5
assert indexes['uid_1']['unique']
assert indexes['time_-1_scan_id_-1']
assert indexes['$**_text']
assert indexes['data_session_1']
indexes = metadatastore_db.run_stop.index_information()
assert len(indexes.keys()) == 5
assert indexes['uid_1']['unique']
assert indexes['run_start_1']['unique']
assert indexes['time_-1']
assert indexes['$**_text']
indexes = metadatastore_db.event_descriptor.index_information()
assert len(indexes.keys()) == 5
assert indexes['uid_1']['unique']
assert indexes['run_start_-1_time_-1']
assert indexes['time_-1']
assert indexes['$**_text']
indexes = metadatastore_db.event.index_information()
assert len(indexes.keys()) == 3
assert indexes['uid_1']['unique']
assert indexes['descriptor_-1_time_1']
def bundle(request, intake_server, example_data, db_factory): # noqa
fullname = os.path.join(TMP_DIR, YAML_FILENAME)
mds_db = db_factory()
assets_db = db_factory()
serializer = Serializer(mds_db, assets_db)
uid, docs = example_data
for name, doc in docs:
serializer(name, doc)
def extract_uri(db):
return f'mongodb://{db.client.address[0]}:{db.client.address[1]}/{db.name}'
with open(fullname, 'w') as f:
f.write(f'''
plugins:
source:
- module: intake_bluesky
sources:
xyz:
description: Some imaginary beamline
driver: intake_bluesky.mongo_normalized.BlueskyMongoCatalog
container: catalog
args:
metadatastore_db: {extract_uri(mds_db)}
asset_registry_db: {extract_uri(assets_db)}
handler_registry:
NPY_SEQ: ophyd.sim.NumpySeqHandler
metadata:
beamline: "00-ID"
''')
time.sleep(2)
if request.param == 'local':
cat = intake.Catalog(os.path.join(TMP_DIR, YAML_FILENAME))
elif request.param == 'remote':
cat = intake.Catalog(intake_server, page_size=10)
else:
raise ValueError
return types.SimpleNamespace(cat=cat, uid=uid, docs=docs)
def init_ingest_service(db: MongoClient):
bluesky_context.serializer = Serializer(metadatastore_db=db, asset_registry_db=db)
bluesky_context.db = db
service_context.db = db
service_context.ingest_jobs = db['ingest_jobs']
service_context.ingest_jobs.create_index(
[
('submit_time', -1)
]
)
service_context.ingest_jobs.create_index(
[
('status', -1)
]
)
service_context.ingest_jobs.create_index(
[
('id', 1),
],
unique=True
)
service_context.ingest_mappings = db['ingest_mappings']
service_context.ingest_mappings.create_index(
[
('name', -1),
('version', -1),
],
unique=True
)
service_context.ingest_mappings.create_index(
[
('id', 1),
],
unique=True
)
def main():
load_dotenv(verbose=True)
input_root = os.getenv('input_root')
output_root = os.getenv('output_root')
msg_pack_dir = os.getenv('msg_pack_dir')
paths = glob.glob(os.path.join(input_root,
os.getenv('input_relative'))+'/**/*.*', recursive=True)
logger.info(paths)
etl_executor = etl.ingest.ETLExecutor(input_root, output_root, tagging_callback)
db = pymongo.MongoClient(
username=os.getenv('tag_db_user'),
password=os.getenv('tag_db_password'),
host=os.getenv('tag_db_host'))
tag_svc = TagService(db, db_name='tagging')
databroker_db_name = os.getenv('tag_databroker_db')
serializer = Serializer(
db[databroker_db_name],
db[databroker_db_name])
now = datetime.datetime.utcnow().replace(tzinfo = datetime.timezone.utc) \
.astimezone().replace(microsecond = 0).isoformat()
tagging_event = {
"model_name": "scattering ingestor",
"md":[
{"key": "date", "value": now}
]
}
# Tagger, Event, and Asset are all hard coded in right now for testing
# purposes. This should not be kept in further versions
tagger_uid = tag_svc.create_tagger({'uid':None, 'type':'model',
'model_name':'hard_code', 'create_time':10, 'accuracy': 0.01})
tagging_event_uid = tag_svc.create_tagging_event({'uid':None,
'tagger_id':tagger_uid, 'run_time':20},tagger_uid)
count = 1
for file_path in paths:
print(file_path)
if os.path.getsize(file_path) = 0 :
continue
try:
raw_metadata, thumb_metadatas, return_metadata = etl_executor.execute(
file_path, [(223, 'jpg'), (223, 'npy')])
except TypeError as e:
logger.error(e)
raise e
except Exception as e:
logger.error(e)
raise e
else:
docs = etl.ingest.createDocument(raw_metadata, output_root,
thumb_metadatas, return_metadata)
for name, doc in docs:
serializer(name, doc)
if name == 'start':
tag_set = make_tag_set(doc, return_metadata)
tag_svc.create_asset_tags(tag_set, tagging_event_uid)
count += 1
def save_spectrum_to_db(metadata, data):
ser = Serializer(uri, uri)
uid = _save_spectrum_to_db(ser, metadata, data)
return uid
def test_validation_error(db_factory, example_data):
metadatastore_db = db_factory()
asset_registry_db = db_factory()
serializer = Serializer(metadatastore_db, asset_registry_db)
with pytest.raises(ValidationError):
assert serializer.update('start', {})
def test_notimplemented_error(db_factory, example_data):
metadatastore_db = db_factory()
asset_registry_db = db_factory()
serializer = Serializer(metadatastore_db, asset_registry_db)
with pytest.raises(NotImplementedError):
assert serializer.update('not_start', {})
def _get_serializer(self):
"This is used internally by v1.Broker. It may be removed in future."
from suitcase.mongo_normalized import Serializer
return Serializer(self._metadatastore_db, self._asset_registry_db)
import bluesky.plans as bp
from bluesky.callbacks.zmq import Publisher
from bluesky.callbacks.best_effort import BestEffortCallback
from databroker._drivers.mongo_normalized import BlueskyMongoCatalog
from bluesky_adaptive.per_start import adaptive_plan
from ophyd.sim import *
RE = RunEngine()
mds = f"mongodb://localhost:27017/databroker-test-{uuid.uuid4()}"
fs = f"mongodb://localhost:27017/databroker-test-{uuid.uuid4()}"
serializer = Serializer(mds, fs)
catalog = BlueskyMongoCatalog(mds, fs)
p = Publisher("127.0.0.1:4567")
bec = BestEffortCallback()
logger = logging.getLogger("databroker")
logger.setLevel("DEBUG")
handler = logging.StreamHandler()
handler.setLevel("DEBUG")
logger.addHandler(handler)
RE.subscribe(serializer)
RE.subscribe(p)
RE.subscribe(bec)
to_brains = Publisher("127.0.0.1:4567", prefix=b"adaptive")
from pymongo import MongoClient
from bluesky.callbacks.zmq import RemoteDispatcher
from suitcase.mongo_normalized import Serializer
# This listens to a lightweight (0MQ-based) message bus for Documents
# and inserts them into MongoDB.
dispatcher = RemoteDispatcher('localhost:5578')
client = MongoClient('localhost:27017')
serializer = Serializer(client['mds'], client['assets'])
print(client.address)
dispatcher.subscribe(serializer)
dispatcher.start()