def complete_material_history(mat): """ Get a list of every single object in the material history, all as dictionaries. This is useful for testing, if we want the context list that can be used to rehydrate an entire material history. :param mat: root material run :return: a list containing every object connected to mat, each a dictionary with all links substituted. """ from gemd.entity.base_entity import BaseEntity import json from gemd.json import dumps, loads from gemd.util.impl import substitute_links result = [] def body(obj: BaseEntity): copy = substitute_links(loads(dumps(obj))) result.append(json.loads(dumps(copy))["context"][0]) recursive_foreach(mat, body, apply_first=False) return result
def register(self, model: ResourceType, dry_run=False): """ Create a new element of the collection or update an existing element. If the input model has an ID that corresponds to an existing object in the database, then that object will be updated. Otherwise a new object will be created. Only the top-level object in `model` itself is written to the database with this method. References to other objects are persisted as links, and the object returned by this method has all instances of data objects replaced by instances of LinkByUid. Registering an object which references other objects does NOT implicitly register those other objects. Rather, those other objects' values are ignored, and the pre-existence of objects with their IDs is asserted before attempting to write `model`. Parameters ---------- model: ResourceType The DataConcepts object. dry_run: bool Whether to actually register the item or run a dry run of the register operation. Dry run is intended to be used for validation. Default: false Returns ------- ResourceType A copy of the registered object as it now exists in the database. """ if self.dataset_id is None: raise RuntimeError( "Must specify a dataset in order to register a data model object." ) path = self._get_path() params = {'dry_run': dry_run} # How do we prepare a citrine-python object to be the json in a POST request? # Right now, that method scrubs out None values and replaces top-level objects with links. # Eventually, we want to replace it with the following: # dumped_data = dumps(loads(dumps(model.dump()))) # This dumps the object to a dictionary (model.dump()), and then to a string (dumps()). # But this string is still nested--because it's a dictionary, GEMDJson.dumps() does not # know how to replace the objects with link-by-uids. loads() converts this string into # nested gemd objects, and then the final dumps() converts that to a json-ready string # in which all of the object references have been replaced with link-by-uids. temp_scope = str(uuid4()) scope = temp_scope if dry_run else CITRINE_SCOPE GEMDJson(scope=scope).dumps( model) # This apparent no-op populates uids dumped_data = replace_objects_with_links(scrub_none(model.dump())) recursive_foreach( model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids data = self.session.post_resource(path, dumped_data, params=params) full_model = self.build(data) return full_model
def register_all(self, models: List[ResourceType], dry_run=False) -> List[ResourceType]: """ [ALPHA] Create or update each model in models. This method has the same behavior as `register`, except that all no models will be written if any one of them is invalid. Using this method should yield significant improvements to write speed over separate calls to `register`. Parameters ---------- models: List[ResourceType] The objects to be written. dry_run: bool Whether to actually register the objects or run a dry run of the register operation. Dry run is intended to be used for validation. Default: false Returns ------- List[ResourceType] Each object model as it now exists in the database. The order and number of models is guaranteed to be the same as originally specified. """ if self.dataset_id is None: raise RuntimeError( "Must specify a dataset in order to register a data model object." ) path = self._get_path() params = {'dry_run': dry_run} temp_scope = str(uuid4()) scope = temp_scope if dry_run else CITRINE_SCOPE json = GEMDJson(scope=scope) [json.dumps(x) for x in models] # This apparent no-op populates uids objects = [ replace_objects_with_links(scrub_none(model.dump())) for model in models ] recursive_foreach( models, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids response_data = self.session.put_resource(path + '/batch', json={'objects': objects}, params=params) return [self.build(obj) for obj in response_data['objects']]
def register(self, model: ResourceType, dry_run=False): """ Create a new element of the collection or update an existing element. If the input model has an ID that corresponds to an existing object in the database, then that object will be updated. Otherwise a new object will be created. Only the top-level object in `model` itself is written to the database with this method. References to other objects are persisted as links, and the object returned by this method has all instances of data objects replaced by instances of LinkByUid. Registering an object which references other objects does NOT implicitly register those other objects. Rather, those other objects' values are ignored, and the pre-existence of objects with their IDs is asserted before attempting to write `model`. Parameters ---------- model: ResourceType The DataConcepts object. dry_run: bool Whether to actually register the item or run a dry run of the register operation. Dry run is intended to be used for validation. Default: false Returns ------- ResourceType A copy of the registered object as it now exists in the database. """ if self.dataset_id is None: raise RuntimeError( "Must specify a dataset in order to register a data model object." ) path = self._get_path() params = {'dry_run': dry_run} temp_scope = str(uuid4()) scope = temp_scope if dry_run else CITRINE_SCOPE GEMDJson(scope=scope).dumps( model) # This apparent no-op populates uids dumped_data = replace_objects_with_links(scrub_none(model.dump())) recursive_foreach( model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids data = self.session.post_resource(path, dumped_data, params=params) full_model = self.build(data) return full_model
def validate_templates(self, model: DataObjectResourceType, object_template: Optional[ObjectTemplateResourceType] = None, ingredient_process_template: Optional[ProcessTemplate] = None)\ -> List[ValidationError]: """ Validate a data object against its templates. Validates against provided object templates (passed in as parameters) and stored attribute templates linked on the data object. :param model: the data object to validate :param object_template: optional object template to validate against :param ingredient_process_template: optional process template to validate ingredient against. Ignored unless data object is an IngredientSpec or IngredientRun. :return: List[ValidationError] of validation errors encountered. Empty if successful. """ path = self._get_path(ignore_dataset=True) + "/validate-templates" temp_scope = str(uuid4()) GEMDJson(scope=temp_scope).dumps( model) # This apparent no-op populates uids dumped_data = replace_objects_with_links(scrub_none(model.dump())) recursive_foreach( model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids request_data = {"dataObject": dumped_data} if object_template is not None: request_data["objectTemplate"] = \ replace_objects_with_links(scrub_none(object_template.dump())) if ingredient_process_template is not None: request_data["ingredientProcessTemplate"] = \ replace_objects_with_links(scrub_none(ingredient_process_template.dump())) try: self.session.put_resource(path, request_data) return [] except BadRequest as e: if e.api_error is not None and e.api_error.validation_errors: return e.api_error.validation_errors raise e
def test_recursive_foreach(): """Test that recursive_foreach() applies a method to every object.""" new_tag = "Extra tag" def func(base_ent): """Adds a specific tag to the object.""" base_ent.tags.extend([new_tag]) return param_template = ParameterTemplate("a param template", bounds=RealBounds(0, 100, '')) meas_template = MeasurementTemplate("Measurement template", parameters=[param_template]) parameter = Parameter(name="A parameter", value=NormalReal(mean=17, std=1, units='')) measurement = MeasurementSpec(name="name", parameters=parameter, template=meas_template) test_dict = {"foo": measurement} recursive_foreach(test_dict, func, apply_first=True) for ent in [param_template, meas_template, measurement]: assert new_tag in ent.tags
def test_cake(): """Create cake, serialize, deserialize.""" cake = make_cake(seed=42) # Check that all the objects show up tot_count = 0 def increment(dummy): nonlocal tot_count tot_count += 1 recursive_foreach(cake, increment) assert tot_count == 139 # Check that no UIDs collide uid_seen = dict() def _check_ids(obj): nonlocal uid_seen for scope in obj.uids: lbl = '{}::{}'.format(scope, obj.uids[scope].lower()) if lbl in uid_seen: assert uid_seen[lbl] == id(obj), "'{}' seen twice".format(lbl) uid_seen[lbl] = id(obj) recursive_foreach(cake, _check_ids) # Check that all recursive and square links are structured correctly def _check_crosslinks(obj): if isinstance(obj, MaterialSpec): assert obj.process.output_material == obj elif isinstance(obj, MaterialRun): assert obj.process.output_material == obj for msr in obj.measurements: assert msr.material == obj assert obj.spec.process == obj.process.spec elif isinstance(obj, ProcessRun): assert obj.output_material.process == obj assert obj.spec.output_material == obj.output_material.spec elif isinstance(obj, ProcessSpec): assert obj.output_material.process == obj elif isinstance(obj, MeasurementSpec): pass # Doesn't link elif isinstance(obj, MeasurementRun): assert obj in obj.material.measurements elif isinstance(obj, IngredientSpec): assert obj in obj.process.ingredients elif isinstance(obj, IngredientRun): assert obj in obj.process.ingredients assert obj.spec.material == obj.material.spec recursive_foreach(cake, _check_crosslinks)
def test_cake(): """Create cake, serialize, deserialize.""" cake = make_cake() def test_for_loss(obj): assert (obj == loads(dumps(obj))) recursive_foreach(cake, test_for_loss) # And verify equality was working in the first place cake2 = loads(dumps(cake)) cake2.name = "It's a trap!" assert (cake2 != cake) cake2.name = cake.name assert (cake == cake2) cake2.uids['new'] = "It's a trap!" assert (cake2 != cake) # Check that all the objects show up tot_count = 0 def increment(dummy): nonlocal tot_count tot_count += 1 recursive_foreach(cake, increment) assert tot_count == 131 # And make sure nothing was lost tot_count = 0 recursive_foreach(loads(dumps(complete_material_history(cake))), increment) assert tot_count == 131 # Check that no UIDs collide uid_seen = dict() def check_ids(obj): nonlocal uid_seen for scope in obj.uids: lbl = '{}::{}'.format(scope, obj.uids[scope]) if lbl in uid_seen: assert uid_seen[lbl] == id(obj) uid_seen[lbl] = id(obj) recursive_foreach(cake, check_ids) queue = [cake] seen = set() while queue: obj = queue.pop() if obj in seen: continue seen.add(obj) if isinstance(obj, MaterialSpec): if obj.process is not None: queue.append(obj.process) assert obj.process.output_material == obj elif isinstance(obj, MaterialRun): if obj.process is not None: queue.append(obj.process) assert obj.process.output_material == obj if obj.measurements: queue.extend(obj.measurements) for msr in obj.measurements: assert msr.material == obj if obj.spec is not None: queue.append(obj.spec) if obj.process is not None: assert obj.spec.process == obj.process.spec elif isinstance(obj, ProcessRun): if obj.ingredients: queue.extend(obj.ingredients) if obj.output_material is not None: queue.append(obj.output_material) assert obj.output_material.process == obj if obj.spec is not None: assert obj.spec.output_material == obj.output_material.spec elif isinstance(obj, ProcessSpec): if obj.ingredients: queue.extend(obj.ingredients) if obj.output_material is not None: queue.append(obj.output_material) assert obj.output_material.process == obj elif isinstance(obj, MeasurementSpec): pass # Doesn't link elif isinstance(obj, MeasurementRun): if obj.spec: queue.append(obj.spec) elif isinstance(obj, IngredientSpec): if obj.material: queue.append(obj.material) elif isinstance(obj, IngredientRun): if obj.spec: queue.append(obj.spec) if obj.material and isinstance(obj.material, MaterialRun): assert obj.spec.material == obj.material.spec if obj.material: queue.append(obj.material)
def async_update(self, model: ResourceType, *, dry_run: bool = False, wait_for_response: bool = True, timeout: float = 2 * 60, polling_delay: float = 1.0) -> Optional[UUID]: """ [ALPHA] Update a particular element of the collection with data validation. Update a particular element of the collection, doing a deeper check to ensure that the dependent data objects are still with the (potentially) changed constraints of this change. This will allow you to make bounds and allowed named/labels changes to templates. Parameters ---------- model: ResourceType The DataConcepts object. dry_run: bool Whether to actually update the item or run a dry run of the update operation. Dry run is intended to be used for validation. Default: false wait_for_response: Whether to poll for the eventual response. This changes the return type (see below). timeout: How long to poll for the result before giving up. This is expressed in (fractional) seconds. polling_delay: How long to delay between each polling retry attempt. Returns ------- Optional[UUID] If wait_for_response if True, then this call will poll the backend, waiting for the eventual job result. In the case of successful validation/update, a return value of None is provided which indicates success. In the case of a failure validating or processing the update, an exception (JobFailureError) is raised and an error message is logged with the underlying reason of the failure. If wait_for_response if False, A job ID (of type UUID) is returned that one can use to poll for the job completion and result with the :func:`~citrine.resources.DataConceptsCollection.poll_async_update_job` method. """ temp_scope = str(uuid4()) GEMDJson(scope=temp_scope).dumps( model) # This apparent no-op populates uids dumped_data = replace_objects_with_links(scrub_none(model.dump())) recursive_foreach( model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids scope = CITRINE_SCOPE id = dumped_data['uids']['id'] if self.dataset_id is None: raise RuntimeError("Must specify a dataset in order to update " "a data model object with data validation.") url = self._get_path() + \ "/" + scope + "/" + id + "/async" response_json = self.session.put_resource(url, dumped_data, params={'dry_run': dry_run}) job_id = response_json["job_id"] if wait_for_response: self.poll_async_update_job(job_id, timeout=timeout, polling_delay=polling_delay) # That worked, nothing returned in this case return None else: # TODO: use JobSubmissionResponse here instead return job_id