def complete_material_history(mat):
"""
Get a list of every single object in the material history, all as dictionaries.
This is useful for testing, if we want the context list that can be used to rehydrate
an entire material history.
:param mat: root material run
:return: a list containing every object connected to mat, each a dictionary with all
links substituted.
"""
from gemd.entity.base_entity import BaseEntity
import json
from gemd.json import dumps, loads
from gemd.util.impl import substitute_links
result = []
def body(obj: BaseEntity):
copy = substitute_links(loads(dumps(obj)))
result.append(json.loads(dumps(copy))["context"][0])
recursive_foreach(mat, body, apply_first=False)
return result
def register(self, model: ResourceType, dry_run=False):
"""
Create a new element of the collection or update an existing element.
If the input model has an ID that corresponds to an existing object in the
database, then that object will be updated. Otherwise a new object will be created.
Only the top-level object in `model` itself is written to the database with this
method. References to other objects are persisted as links, and the object returned
by this method has all instances of data objects replaced by instances of LinkByUid.
Registering an object which references other objects does NOT implicitly register
those other objects. Rather, those other objects' values are ignored, and the
pre-existence of objects with their IDs is asserted before attempting to write
`model`.
Parameters
----------
model: ResourceType
The DataConcepts object.
dry_run: bool
Whether to actually register the item or run a dry run of the register operation.
Dry run is intended to be used for validation. Default: false
Returns
-------
ResourceType
A copy of the registered object as it now exists in the database.
"""
if self.dataset_id is None:
raise RuntimeError(
"Must specify a dataset in order to register a data model object."
)
path = self._get_path()
params = {'dry_run': dry_run}
# How do we prepare a citrine-python object to be the json in a POST request?
# Right now, that method scrubs out None values and replaces top-level objects with links.
# Eventually, we want to replace it with the following:
# dumped_data = dumps(loads(dumps(model.dump())))
# This dumps the object to a dictionary (model.dump()), and then to a string (dumps()).
# But this string is still nested--because it's a dictionary, GEMDJson.dumps() does not
# know how to replace the objects with link-by-uids. loads() converts this string into
# nested gemd objects, and then the final dumps() converts that to a json-ready string
# in which all of the object references have been replaced with link-by-uids.
temp_scope = str(uuid4())
scope = temp_scope if dry_run else CITRINE_SCOPE
GEMDJson(scope=scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
data = self.session.post_resource(path, dumped_data, params=params)
full_model = self.build(data)
return full_model
def register_all(self,
models: List[ResourceType],
dry_run=False) -> List[ResourceType]:
"""
[ALPHA] Create or update each model in models.
This method has the same behavior as `register`, except that all no models will be
written if any one of them is invalid.
Using this method should yield significant improvements to write speed over separate
calls to `register`.
Parameters
----------
models: List[ResourceType]
The objects to be written.
dry_run: bool
Whether to actually register the objects or run a dry run of the register operation.
Dry run is intended to be used for validation. Default: false
Returns
-------
List[ResourceType]
Each object model as it now exists in the database. The order and number of models
is guaranteed to be the same as originally specified.
"""
if self.dataset_id is None:
raise RuntimeError(
"Must specify a dataset in order to register a data model object."
)
path = self._get_path()
params = {'dry_run': dry_run}
temp_scope = str(uuid4())
scope = temp_scope if dry_run else CITRINE_SCOPE
json = GEMDJson(scope=scope)
[json.dumps(x) for x in models] # This apparent no-op populates uids
objects = [
replace_objects_with_links(scrub_none(model.dump()))
for model in models
]
recursive_foreach(
models, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
response_data = self.session.put_resource(path + '/batch',
json={'objects': objects},
params=params)
return [self.build(obj) for obj in response_data['objects']]
def register(self, model: ResourceType, dry_run=False):
"""
Create a new element of the collection or update an existing element.
If the input model has an ID that corresponds to an existing object in the
database, then that object will be updated. Otherwise a new object will be created.
Only the top-level object in `model` itself is written to the database with this
method. References to other objects are persisted as links, and the object returned
by this method has all instances of data objects replaced by instances of LinkByUid.
Registering an object which references other objects does NOT implicitly register
those other objects. Rather, those other objects' values are ignored, and the
pre-existence of objects with their IDs is asserted before attempting to write
`model`.
Parameters
----------
model: ResourceType
The DataConcepts object.
dry_run: bool
Whether to actually register the item or run a dry run of the register operation.
Dry run is intended to be used for validation. Default: false
Returns
-------
ResourceType
A copy of the registered object as it now exists in the database.
"""
if self.dataset_id is None:
raise RuntimeError(
"Must specify a dataset in order to register a data model object."
)
path = self._get_path()
params = {'dry_run': dry_run}
temp_scope = str(uuid4())
scope = temp_scope if dry_run else CITRINE_SCOPE
GEMDJson(scope=scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
data = self.session.post_resource(path, dumped_data, params=params)
full_model = self.build(data)
return full_model
def validate_templates(self,
model: DataObjectResourceType,
object_template: Optional[ObjectTemplateResourceType] = None,
ingredient_process_template: Optional[ProcessTemplate] = None)\
-> List[ValidationError]:
"""
Validate a data object against its templates.
Validates against provided object templates (passed in as parameters) and stored attribute
templates linked on the data object.
:param model: the data object to validate
:param object_template: optional object template to validate against
:param ingredient_process_template: optional process template to validate ingredient
against. Ignored unless data object is an IngredientSpec or IngredientRun.
:return: List[ValidationError] of validation errors encountered. Empty if successful.
"""
path = self._get_path(ignore_dataset=True) + "/validate-templates"
temp_scope = str(uuid4())
GEMDJson(scope=temp_scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
request_data = {"dataObject": dumped_data}
if object_template is not None:
request_data["objectTemplate"] = \
replace_objects_with_links(scrub_none(object_template.dump()))
if ingredient_process_template is not None:
request_data["ingredientProcessTemplate"] = \
replace_objects_with_links(scrub_none(ingredient_process_template.dump()))
try:
self.session.put_resource(path, request_data)
return []
except BadRequest as e:
if e.api_error is not None and e.api_error.validation_errors:
return e.api_error.validation_errors
raise e
def test_recursive_foreach():
"""Test that recursive_foreach() applies a method to every object."""
new_tag = "Extra tag"
def func(base_ent):
"""Adds a specific tag to the object."""
base_ent.tags.extend([new_tag])
return
param_template = ParameterTemplate("a param template",
bounds=RealBounds(0, 100, ''))
meas_template = MeasurementTemplate("Measurement template",
parameters=[param_template])
parameter = Parameter(name="A parameter",
value=NormalReal(mean=17, std=1, units=''))
measurement = MeasurementSpec(name="name",
parameters=parameter,
template=meas_template)
test_dict = {"foo": measurement}
recursive_foreach(test_dict, func, apply_first=True)
for ent in [param_template, meas_template, measurement]:
assert new_tag in ent.tags
def test_cake():
"""Create cake, serialize, deserialize."""
cake = make_cake(seed=42)
# Check that all the objects show up
tot_count = 0
def increment(dummy):
nonlocal tot_count
tot_count += 1
recursive_foreach(cake, increment)
assert tot_count == 139
# Check that no UIDs collide
uid_seen = dict()
def _check_ids(obj):
nonlocal uid_seen
for scope in obj.uids:
lbl = '{}::{}'.format(scope, obj.uids[scope].lower())
if lbl in uid_seen:
assert uid_seen[lbl] == id(obj), "'{}' seen twice".format(lbl)
uid_seen[lbl] = id(obj)
recursive_foreach(cake, _check_ids)
# Check that all recursive and square links are structured correctly
def _check_crosslinks(obj):
if isinstance(obj, MaterialSpec):
assert obj.process.output_material == obj
elif isinstance(obj, MaterialRun):
assert obj.process.output_material == obj
for msr in obj.measurements:
assert msr.material == obj
assert obj.spec.process == obj.process.spec
elif isinstance(obj, ProcessRun):
assert obj.output_material.process == obj
assert obj.spec.output_material == obj.output_material.spec
elif isinstance(obj, ProcessSpec):
assert obj.output_material.process == obj
elif isinstance(obj, MeasurementSpec):
pass # Doesn't link
elif isinstance(obj, MeasurementRun):
assert obj in obj.material.measurements
elif isinstance(obj, IngredientSpec):
assert obj in obj.process.ingredients
elif isinstance(obj, IngredientRun):
assert obj in obj.process.ingredients
assert obj.spec.material == obj.material.spec
recursive_foreach(cake, _check_crosslinks)
def test_cake():
"""Create cake, serialize, deserialize."""
cake = make_cake()
def test_for_loss(obj):
assert (obj == loads(dumps(obj)))
recursive_foreach(cake, test_for_loss)
# And verify equality was working in the first place
cake2 = loads(dumps(cake))
cake2.name = "It's a trap!"
assert (cake2 != cake)
cake2.name = cake.name
assert (cake == cake2)
cake2.uids['new'] = "It's a trap!"
assert (cake2 != cake)
# Check that all the objects show up
tot_count = 0
def increment(dummy):
nonlocal tot_count
tot_count += 1
recursive_foreach(cake, increment)
assert tot_count == 131
# And make sure nothing was lost
tot_count = 0
recursive_foreach(loads(dumps(complete_material_history(cake))), increment)
assert tot_count == 131
# Check that no UIDs collide
uid_seen = dict()
def check_ids(obj):
nonlocal uid_seen
for scope in obj.uids:
lbl = '{}::{}'.format(scope, obj.uids[scope])
if lbl in uid_seen:
assert uid_seen[lbl] == id(obj)
uid_seen[lbl] = id(obj)
recursive_foreach(cake, check_ids)
queue = [cake]
seen = set()
while queue:
obj = queue.pop()
if obj in seen:
continue
seen.add(obj)
if isinstance(obj, MaterialSpec):
if obj.process is not None:
queue.append(obj.process)
assert obj.process.output_material == obj
elif isinstance(obj, MaterialRun):
if obj.process is not None:
queue.append(obj.process)
assert obj.process.output_material == obj
if obj.measurements:
queue.extend(obj.measurements)
for msr in obj.measurements:
assert msr.material == obj
if obj.spec is not None:
queue.append(obj.spec)
if obj.process is not None:
assert obj.spec.process == obj.process.spec
elif isinstance(obj, ProcessRun):
if obj.ingredients:
queue.extend(obj.ingredients)
if obj.output_material is not None:
queue.append(obj.output_material)
assert obj.output_material.process == obj
if obj.spec is not None:
assert obj.spec.output_material == obj.output_material.spec
elif isinstance(obj, ProcessSpec):
if obj.ingredients:
queue.extend(obj.ingredients)
if obj.output_material is not None:
queue.append(obj.output_material)
assert obj.output_material.process == obj
elif isinstance(obj, MeasurementSpec):
pass # Doesn't link
elif isinstance(obj, MeasurementRun):
if obj.spec:
queue.append(obj.spec)
elif isinstance(obj, IngredientSpec):
if obj.material:
queue.append(obj.material)
elif isinstance(obj, IngredientRun):
if obj.spec:
queue.append(obj.spec)
if obj.material and isinstance(obj.material, MaterialRun):
assert obj.spec.material == obj.material.spec
if obj.material:
queue.append(obj.material)
def async_update(self,
model: ResourceType,
*,
dry_run: bool = False,
wait_for_response: bool = True,
timeout: float = 2 * 60,
polling_delay: float = 1.0) -> Optional[UUID]:
"""
[ALPHA] Update a particular element of the collection with data validation.
Update a particular element of the collection, doing a deeper check to ensure that
the dependent data objects are still with the (potentially) changed constraints
of this change. This will allow you to make bounds and allowed named/labels changes
to templates.
Parameters
----------
model: ResourceType
The DataConcepts object.
dry_run: bool
Whether to actually update the item or run a dry run of the update operation.
Dry run is intended to be used for validation. Default: false
wait_for_response:
Whether to poll for the eventual response. This changes the return type (see
below).
timeout:
How long to poll for the result before giving up. This is expressed in
(fractional) seconds.
polling_delay:
How long to delay between each polling retry attempt.
Returns
-------
Optional[UUID]
If wait_for_response if True, then this call will poll the backend, waiting
for the eventual job result. In the case of successful validation/update,
a return value of None is provided which indicates success. In the case of
a failure validating or processing the update, an exception (JobFailureError)
is raised and an error message is logged with the underlying reason of the
failure.
If wait_for_response if False, A job ID (of type UUID) is returned that one
can use to poll for the job completion and result with the
:func:`~citrine.resources.DataConceptsCollection.poll_async_update_job`
method.
"""
temp_scope = str(uuid4())
GEMDJson(scope=temp_scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
scope = CITRINE_SCOPE
id = dumped_data['uids']['id']
if self.dataset_id is None:
raise RuntimeError("Must specify a dataset in order to update "
"a data model object with data validation.")
url = self._get_path() + \
"/" + scope + "/" + id + "/async"
response_json = self.session.put_resource(url,
dumped_data,
params={'dry_run': dry_run})
job_id = response_json["job_id"]
if wait_for_response:
self.poll_async_update_job(job_id,
timeout=timeout,
polling_delay=polling_delay)
# That worked, nothing returned in this case
return None
else:
# TODO: use JobSubmissionResponse here instead
return job_id