def validate(self, cid, data, object_type=None, object_name=None, needs_err_details=False, _validate=js_validate):
# type: (str, object, str, str, Callable) -> Result
# Result we will return
result = Result()
result.cid = cid
object_type = object_type or self.config.object_type
object_name or self.config.object_name
needs_err_details = needs_err_details or self.config.needs_err_details
try:
js_validate(data, self.config.schema, self.config.validator)
except JSValidationError as e:
# These will be always used, no matter the object/channel type
result.is_ok = False
result.object_type = object_type
result.needs_err_details = needs_err_details
result.error_msg = str(e)
# This is applicable only to JSON-RPC
if object_type == CHANNEL.JSON_RPC:
result.error_extra = {'json_rpc_id': data.get('id')}
else:
result.is_ok = True
return result
def update(self, val: str):
'''
Update string schema type with a string like val.
'''
cname = self.__class__.__name__
if isinstance(val, self.__class__):
self._value = val.pod()
return
if not isinstance(val, str):
raise ValueError(f'illegal type for string {cname}: {type(val)}')
ost = self.ost
schema = dict(type="string")
if ost["pattern"]:
schema["pattern"] = ost["pattern"]
if ost["format"]:
schema["format"] = ost["format"]
from jsonschema import validate as js_validate
from jsonschema import draft7_format_checker
from jsonschema.exceptions import ValidationError
try:
js_validate(instance=val,
schema=schema,
format_checker=draft7_format_checker)
except ValidationError as verr:
raise ValueError(f'format mismatch for string {cname}') from verr
self._value = val
def insert_datum(resource, datum_id, datum_kwargs=None):
"""
Parameters
----------
resource : Resource or Resource.id
Resource object
datum_id : str
Unique identifier for this datum. This is the value stored in
metadatastore and is the value passed to `retrieve` to get
the data back out.
datum_kwargs : dict
dict with any kwargs needed to retrieve this specific datum from the
resource.
"""
try:
spec = resource.spec
except AttributeError:
resource = Resource.objects.get(id=resource)
spec = resource.spec
if datum_kwargs is None:
datum_kwargs = {}
if spec in known_spec:
js_validate(datum_kwargs, known_spec[spec]["datum"])
datum = Datum(resource=resource.id, datum_id=datum_id, datum_kwargs=datum_kwargs)
datum.save(validate=True, write_concern={"w": 1})
return datum
def insert_resource(spec, resource_path, resource_kwargs=None):
"""
Parameters
----------
spec : str
spec used to determine what handler to use to open this
resource.
resource_path : str or None
Url to the physical location of this resource
resource_kwargs : dict
resource_kwargs name/value pairs of additional kwargs to be
passed to the handler to open this resource.
"""
if resource_path is None:
resource_path = ""
if resource_kwargs is None:
resource_kwargs = {}
if spec in known_spec:
js_validate(resource_kwargs, known_spec[spec]["resource"])
resource_object = Resource(spec=spec, resource_path=resource_path, resource_kwargs=resource_kwargs)
resource_object.save(validate=True, write_concern={"w": 1})
return resource_object
def insert_datum(col,
resource,
datum_id,
datum_kwargs,
known_spec,
resource_col,
ignore_duplicate_error=False,
duplicate_exc=None):
if ignore_duplicate_error:
assert duplicate_exc is not None
if duplicate_exc is None:
class _PrivateException(Exception):
pass
duplicate_exc = _PrivateException
try:
resource['spec']
spec = resource['spec']
if spec in known_spec:
js_validate(datum_kwargs, known_spec[spec]['datum'])
except (AttributeError, TypeError):
pass
resource_uid = doc_or_uid_to_uid(resource)
datum = dict(resource=resource_uid,
datum_id=str(datum_id),
datum_kwargs=dict(datum_kwargs))
apply_to_dict_recursively(datum, sanitize_np)
# We are transitioning from ophyd objects inserting directly into a
# Registry to ophyd objects passing documents to the RunEngine which in
# turn inserts them into a Registry. During the transition period, we allow
# an ophyd object to attempt BOTH so that configuration files are
# compatible with both the new model and the old model. Thus, we need to
# ignore the second attempt to insert.
try:
col.insert_one(datum)
except duplicate_exc:
if ignore_duplicate_error:
warnings.warn("Ignoring attempt to insert Resource with duplicate "
"uid, assuming that both ophyd and bluesky "
"attempted to insert this document. Remove the "
"Registry (`reg` parameter) from your ophyd "
"instance to remove this warning.")
else:
raise
# do not leak mongo objectID
datum.pop('_id', None)
return datum
def insert_resource(col,
spec,
resource_path,
resource_kwargs,
known_spec,
root,
path_semantics='posix',
uid=None,
run_start=None,
ignore_duplicate_error=False,
duplicate_exc=None):
if ignore_duplicate_error:
assert duplicate_exc is not None
resource_kwargs = dict(resource_kwargs)
if spec in known_spec:
js_validate(resource_kwargs, known_spec[spec]['resource'])
if uid is None:
uid = str(uuid.uuid4())
resource_object = dict(spec=str(spec),
resource_path=str(resource_path),
root=str(root),
resource_kwargs=resource_kwargs,
path_semantics=path_semantics,
uid=uid)
# This is special-cased because it was added later.
# Someday this may be required and no longer special-cased.
if run_start is not None:
resource_object['run_start'] = run_start
# We are transitioning from ophyd objects inserting directly into a
# Registry to ophyd objects passing documents to the RunEngine which in
# turn inserts them into a Registry. During the transition period, we allow
# an ophyd object to attempt BOTH so that configuration files are
# compatible with both the new model and the old model. Thus, we need to
# ignore the second attempt to insert.
try:
col.insert_one(resource_object)
except duplicate_exc:
if ignore_duplicate_error:
warnings.warn("Ignoring attempt to insert Datum with duplicate "
"datum_id, assuming that both ophyd and bluesky "
"attempted to insert this document. Remove the "
"Registry (`reg` parameter) from your ophyd "
"instance to remove this warning.")
else:
raise
resource_object['id'] = resource_object['uid']
resource_object.pop('_id', None)
return resource_object
def insert_resource(col, spec, resource_path, resource_kwargs,
known_spec, root=''):
resource_kwargs = dict(resource_kwargs)
if spec in known_spec:
js_validate(resource_kwargs, known_spec[spec]['resource'])
if root:
resource_path = os.path.join(root, resource_path)
resource_object = dict(spec=str(spec),
resource_path=str(resource_path),
resource_kwargs=resource_kwargs)
col.insert_one(resource_object)
# rename to play nice with ME
resource_object['id'] = resource_object.pop('_id')
return resource_object
def insert_resource(col, spec, resource_path, resource_kwargs, known_spec,
root):
resource_kwargs = dict(resource_kwargs)
if spec in known_spec:
js_validate(resource_kwargs, known_spec[spec]['resource'])
resource_object = dict(spec=str(spec),
resource_path=str(resource_path),
root=str(root),
resource_kwargs=resource_kwargs,
uid=str(uuid.uuid4()))
col.insert_one(resource_object)
# maintain back compatibility
resource_object['id'] = resource_object['uid']
resource_object.pop('_id', None)
return Document('resource', resource_object)
def insert_resource(col, spec, resource_path, resource_kwargs,
known_spec, root):
resource_kwargs = dict(resource_kwargs)
if spec in known_spec:
js_validate(resource_kwargs, known_spec[spec]['resource'])
resource_object = dict(spec=str(spec),
resource_path=str(resource_path),
root=str(root),
resource_kwargs=resource_kwargs,
uid=str(uuid.uuid4()))
col.insert_one(resource_object)
# maintain back compatibility
resource_object['id'] = resource_object['uid']
resource_object.pop('_id', None)
return Document('resource', resource_object)
def insert_datum(col, resource, datum_id, datum_kwargs, known_spec,
resource_col):
try:
resource['spec']
except (AttributeError, TypeError):
resource = resource_col.find_one({'_id': ObjectId(resource)})
resource['id'] = resource['_id']
spec = resource['spec']
if spec in known_spec:
js_validate(datum_kwargs, known_spec[spec]['datum'])
datum = dict(resource=ObjectId(resource['id']),
datum_id=str(datum_id),
datum_kwargs=dict(datum_kwargs))
col.insert_one(datum)
# do not leak mongo objectID
datum.pop('_id', None)
return Document('datum', datum)
def insert_datum(col, resource, datum_id, datum_kwargs, known_spec,
resource_col):
try:
resource['spec']
except (AttributeError, TypeError):
resource = resource_col.find_one({'uid': resource})
spec = resource['spec']
if spec in known_spec:
js_validate(datum_kwargs, known_spec[spec]['datum'])
datum = dict(resource=resource['uid'],
datum_id=str(datum_id),
datum_kwargs=dict(datum_kwargs))
col.insert_one(datum)
# do not leak mongo objectID
datum.pop('_id', None)
return Document('datum', datum)
def validate(model, schema, validator="jsonschema"):
'Validate model against schema with validator'
if validator == "jsonschema":
from jsonschema import validate as js_validate
from jsonschema import draft7_format_checker
return js_validate(instance=model,
schema=schema,
format_checker=draft7_format_checker)
if validator == "fastjsonschema":
from fastjsonschema import validate as fjs_validate
return fjs_validate(schema, model)
raise ValueError(f"unknown validator: {validator}")
def wash_string(types, ost, *args, **kwds):
name = ost["name"]
if not args:
raise ValueError(f'illegal string setting for {name}')
val = args[0]
if not isinstance(val, str):
raise ValueError(f'illegal string {name} value: {val}')
# let jsonschema do heavy lifting
schema = dict(type="string")
if "format" in ost:
schema["format"] = ost["format"]
if "pattern" in ost:
schema["pattern"] = ost["pattern"]
try:
js_validate(instance=val,
schema=schema,
format_checker=draft7_format_checker)
except ValidationError as verr:
raise ValueError(
f'illegal string format {name} value: {val}') from verr
return val
def _register_resource(self, col, uid, spec, root, rpath, rkwargs,
path_semantics):
run_start = None
ignore_duplicate_error = False
duplicate_exc = None
if root is None:
root = ''
resource_kwargs = dict(rkwargs)
if spec in self.known_spec:
js_validate(resource_kwargs, self.known_spec[spec]['resource'])
resource_object = dict(spec=str(spec),
resource_path=str(rpath),
root=str(root),
resource_kwargs=resource_kwargs,
path_semantics=path_semantics,
uid=uid)
try:
col.insert_one(resource_object)
except duplicate_exc:
if ignore_duplicate_error:
warnings.warn(
"Ignoring attempt to insert Datum with duplicate "
"datum_id, assuming that both ophyd and bluesky "
"attempted to insert this document. Remove the "
"Registry (`reg` parameter) from your ophyd "
"instance to remove this warning.")
else:
raise
resource_object['id'] = resource_object['uid']
resource_object.pop('_id', None)
ret = resource_object['uid']
return ret
def insert_resource(col, spec, resource_path, resource_kwargs,
known_spec, root, path_semantics='posix', uid=None,
run_start=None, id=None,
ignore_duplicate_error=False, duplicate_exc=None):
"""Insert resource into a databroker.
Parameters
----------
col : pymongo.Collection instance
Collection to insert data into
spec : str
The resource data spec
resource_path : str
The path to the resource files
resource_kwargs : dict
The kwargs for the resource
known_spec : set
The known specs
root : str
The root of the file path
path_semantics : str, optional
The name of the path semantics, e.g. ``posix`` for Linux systems
uid : str, optional
The unique ID for the resource
run_start : str, optional
The unique ID for the start document the resource is associated with
id : str, optional
Dummy variable so that we round trip resources, same as ``uid``
Returns
-------
resource_object : dict
The resource
"""
if ignore_duplicate_error:
assert duplicate_exc is not None
if duplicate_exc is None:
class _PrivateException(Exception):
pass
duplicate_exc = _PrivateException
resource_kwargs = dict(resource_kwargs)
if spec in known_spec:
js_validate(resource_kwargs, known_spec[spec]['resource'])
if uid is None:
uid = str(uuid.uuid4())
resource_object = dict(spec=str(spec),
resource_path=str(resource_path),
root=str(root),
resource_kwargs=resource_kwargs,
path_semantics=path_semantics,
uid=uid)
# This is special-cased because it was added later.
# Someday this may be required and no longer special-cased.
if run_start is not None:
resource_object['run_start'] = run_start
# We are transitioning from ophyd objects inserting directly into a
# Registry to ophyd objects passing documents to the RunEngine which in
# turn inserts them into a Registry. During the transition period, we allow
# an ophyd object to attempt BOTH so that configuration files are
# compatible with both the new model and the old model. Thus, we need to
# ignore the second attempt to insert.
try:
col.insert_one(resource_object)
except duplicate_exc:
if ignore_duplicate_error:
warnings.warn("Ignoring attempt to insert Datum with duplicate "
"datum_id, assuming that both ophyd and bluesky "
"attempted to insert this document. Remove the "
"Registry (`reg` parameter) from your ophyd "
"instance to remove this warning.")
else:
raise
resource_object['id'] = resource_object['uid']
resource_object.pop('_id', None)
return resource_object