def test_validate_templates_successful_all_params(collection, session):
"""
Test that DataObjectCollection.validate_templates() handles a successful return value when
passing in all params
"""
# Given
project_id = '6b608f78-e341-422c-8076-35adc8828545'
run = MaterialRunFactory(name="validate_templates_successful")
template = MaterialTemplateFactory()
unused_process_template = ProcessTemplateFactory()
# When
session.set_response("")
errors = collection.validate_templates(run, template,
unused_process_template)
# Then
assert 1 == session.num_calls
expected_call = FakeCall(
method="PUT",
path="projects/{}/material-runs/validate-templates".format(project_id),
json={
"dataObject": scrub_none(run.dump()),
"objectTemplate": scrub_none(template.dump()),
"ingredientProcessTemplate":
scrub_none(unused_process_template.dump())
})
assert session.last_call == expected_call
assert errors == []
def test_validate_templates_errors(collection, session):
"""
Test that DataObjectCollection.validate_templates() handles validation errors
"""
# Given
project_id = '6b608f78-e341-422c-8076-35adc8828545'
run = MaterialRunFactory(name="")
# When
validation_error = ValidationError(failure_message="you failed",
failure_id="failure_id")
session.set_response(
BadRequest(
"path",
FakeRequestResponseApiError(400, "Bad Request",
[validation_error])))
errors = collection.validate_templates(run)
# Then
assert 1 == session.num_calls
expected_call = FakeCall(
method="PUT",
path="projects/{}/material-runs/validate-templates".format(project_id),
json={"dataObject": scrub_none(run.dump())})
assert session.last_call == expected_call
assert errors == [validation_error]
def register(self, model: Dataset) -> Dataset:
"""
Create a new element of the collection by registering an existing resource.
This differs from super().register() in that None fields are scrubbed, and the json
response is not assumed to come in a dictionary with a single entry 'dataset'.
Both of these behaviors are in contrast to the behavior of projects. Eventually they
will be unified in the backend, and one register() method will suffice.
Parameters
----------
model: Dataset
The dataset to register.
Returns
-------
Dataset
A copy of the registered dataset as it now exists in the database.
"""
path = self._get_path()
dumped_dataset = model.dump()
dumped_dataset["deleted"] = None
data = self.session.post_resource(path, scrub_none(dumped_dataset))
full_model = self.build(data)
full_model.project_id = self.project_id
return full_model
def register(self, model: ResourceType):
"""
Create a new element of the collection or update an existing element.
If the input model has an ID that corresponds to an existing object in the
database, then that object will be updated. Otherwise a new object will be created.
Parameters
----------
model: DataConcepts
The DataConcepts object.
Returns
-------
DataConcepts
A copy of the registered object as it now exists in the database.
"""
if self.dataset_id is None:
raise RuntimeError("Must specify a dataset in order to register a data model object.")
path = self._get_path()
# How do we prepare a citrine-python object to be the json in a POST request?
# Right now, that method scrubs out None values and replaces top-level objects with links.
# Eventually, we want to replace it with the following:
# dumped_data = dumps(loads(dumps(model.dump())))
# This dumps the object to a dictionary (model.dump()), and then to a string (dumps()).
# But this string is still nested--because it's a dictionary, taurus.dumps() does not know
# how to replace the objects with link-by-uids. loads() converts this string into nested
# taurus objects, and then the final dumps() converts that to a json-ready string in which
# all of the object references have been replaced with link-by-uids.
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
data = self.session.post_resource(path, dumped_data)
full_model = self.build(data)
model.session = self.session
return full_model
def register(self, model: Dataset) -> Dataset:
"""
Create a new dataset in the collection, or update an existing one.
If the Dataset has an ID present, then we update the existing resource,
else we create a new one.
This differs from super().register() in that None fields are scrubbed, and the json
response is not assumed to come in a dictionary with a single entry 'dataset'.
Both of these behaviors are in contrast to the behavior of projects. Eventually they
will be unified in the backend, and one register() method will suffice.
Parameters
----------
model: Dataset
The dataset to register.
Returns
-------
Dataset
A copy of the registered dataset as it now exists in the database.
"""
path = self._get_path()
dumped_dataset = model.dump()
dumped_dataset["deleted"] = None
# Only use the idempotent put approach if a) a unique name is provided, and b)
# the session is configured to use it (default to False for backwards compatibility).
if model.unique_name is not None and self.session.use_idempotent_dataset_put:
# Leverage the create-or-update endpoint if we've got a unique name
data = self.session.put_resource(path, scrub_none(dumped_dataset))
else:
if model.uid is None:
# POST to create a new one if a UID is not assigned
data = self.session.post_resource(path,
scrub_none(dumped_dataset))
else:
# Otherwise PUT to update it
data = self.session.put_resource(self._get_path(model.uid),
scrub_none(dumped_dataset))
full_model = self.build(data)
full_model.project_id = self.project_id
return full_model
def register(self, model: ResourceType, dry_run=False):
"""
Create a new element of the collection or update an existing element.
If the input model has an ID that corresponds to an existing object in the
database, then that object will be updated. Otherwise a new object will be created.
Only the top-level object in `model` itself is written to the database with this
method. References to other objects are persisted as links, and the object returned
by this method has all instances of data objects replaced by instances of LinkByUid.
Registering an object which references other objects does NOT implicitly register
those other objects. Rather, those other objects' values are ignored, and the
pre-existence of objects with their IDs is asserted before attempting to write
`model`.
Parameters
----------
model: ResourceType
The DataConcepts object.
dry_run: bool
Whether to actually register the item or run a dry run of the register operation.
Dry run is intended to be used for validation. Default: false
Returns
-------
ResourceType
A copy of the registered object as it now exists in the database.
"""
if self.dataset_id is None:
raise RuntimeError(
"Must specify a dataset in order to register a data model object."
)
path = self._get_path()
params = {'dry_run': dry_run}
# How do we prepare a citrine-python object to be the json in a POST request?
# Right now, that method scrubs out None values and replaces top-level objects with links.
# Eventually, we want to replace it with the following:
# dumped_data = dumps(loads(dumps(model.dump())))
# This dumps the object to a dictionary (model.dump()), and then to a string (dumps()).
# But this string is still nested--because it's a dictionary, GEMDJson.dumps() does not
# know how to replace the objects with link-by-uids. loads() converts this string into
# nested gemd objects, and then the final dumps() converts that to a json-ready string
# in which all of the object references have been replaced with link-by-uids.
temp_scope = str(uuid4())
scope = temp_scope if dry_run else CITRINE_SCOPE
GEMDJson(scope=scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
data = self.session.post_resource(path, dumped_data, params=params)
full_model = self.build(data)
return full_model
def test_scrub_none():
"""Test that scrub_none() when applied to some examples yields expected results."""
json = dict(key1=1, key2=None)
scrub_none(json)
assert json == dict(key1=1)
json = dict(key1=dict(key11='foo', key12=None),
key2=[
dict(key21=None, key22=17),
dict(key23=None),
dict(key24=34, key25=51)
])
scrub_none(json)
assert json == dict(
key1=dict(key11='foo'),
key2=[dict(key22=17), dict(),
dict(key24=34, key25=51)])
json = dict(key1=1,
key2=[None, 'foo', None, None, 'bar', None],
key3=[None, None, None])
scrub_none(json)
# None should not be removed from lists
assert json == dict(key1=1,
key2=[None, 'foo', None, None, 'bar', None],
key3=[None, None, None])
def validate_templates(self,
model: DataObjectResourceType,
object_template: Optional[ObjectTemplateResourceType] = None,
ingredient_process_template: Optional[ProcessTemplate] = None)\
-> List[ValidationError]:
"""
Validate a data object against its templates.
Validates against provided object templates (passed in as parameters) and stored attribute
templates linked on the data object.
:param model: the data object to validate
:param object_template: optional object template to validate against
:param ingredient_process_template: optional process template to validate ingredient
against. Ignored unless data object is an IngredientSpec or IngredientRun.
:return: List[ValidationError] of validation errors encountered. Empty if successful.
"""
path = self._get_path(ignore_dataset=True) + "/validate-templates"
temp_scope = str(uuid4())
GEMDJson(scope=temp_scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
request_data = {"dataObject": dumped_data}
if object_template is not None:
request_data["objectTemplate"] = \
replace_objects_with_links(scrub_none(object_template.dump()))
if ingredient_process_template is not None:
request_data["ingredientProcessTemplate"] = \
replace_objects_with_links(scrub_none(ingredient_process_template.dump()))
try:
self.session.put_resource(path, request_data)
return []
except BadRequest as e:
if e.api_error is not None and e.api_error.validation_errors:
return e.api_error.validation_errors
raise e
def register_all(self,
models: List[ResourceType],
dry_run=False) -> List[ResourceType]:
"""
[ALPHA] Create or update each model in models.
This method has the same behavior as `register`, except that all no models will be
written if any one of them is invalid.
Using this method should yield significant improvements to write speed over separate
calls to `register`.
Parameters
----------
models: List[ResourceType]
The objects to be written.
dry_run: bool
Whether to actually register the objects or run a dry run of the register operation.
Dry run is intended to be used for validation. Default: false
Returns
-------
List[ResourceType]
Each object model as it now exists in the database. The order and number of models
is guaranteed to be the same as originally specified.
"""
if self.dataset_id is None:
raise RuntimeError(
"Must specify a dataset in order to register a data model object."
)
path = self._get_path()
params = {'dry_run': dry_run}
temp_scope = str(uuid4())
scope = temp_scope if dry_run else CITRINE_SCOPE
json = GEMDJson(scope=scope)
[json.dumps(x) for x in models] # This apparent no-op populates uids
objects = [
replace_objects_with_links(scrub_none(model.dump()))
for model in models
]
recursive_foreach(
models, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
response_data = self.session.put_resource(path + '/batch',
json={'objects': objects},
params=params)
return [self.build(obj) for obj in response_data['objects']]
def register(self, model: ResourceType, dry_run=False):
"""
Create a new element of the collection or update an existing element.
If the input model has an ID that corresponds to an existing object in the
database, then that object will be updated. Otherwise a new object will be created.
Only the top-level object in `model` itself is written to the database with this
method. References to other objects are persisted as links, and the object returned
by this method has all instances of data objects replaced by instances of LinkByUid.
Registering an object which references other objects does NOT implicitly register
those other objects. Rather, those other objects' values are ignored, and the
pre-existence of objects with their IDs is asserted before attempting to write
`model`.
Parameters
----------
model: ResourceType
The DataConcepts object.
dry_run: bool
Whether to actually register the item or run a dry run of the register operation.
Dry run is intended to be used for validation. Default: false
Returns
-------
ResourceType
A copy of the registered object as it now exists in the database.
"""
if self.dataset_id is None:
raise RuntimeError(
"Must specify a dataset in order to register a data model object."
)
path = self._get_path()
params = {'dry_run': dry_run}
temp_scope = str(uuid4())
scope = temp_scope if dry_run else CITRINE_SCOPE
GEMDJson(scope=scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
data = self.session.post_resource(path, dumped_data, params=params)
full_model = self.build(data)
return full_model
def async_update(self,
model: ResourceType,
*,
dry_run: bool = False,
wait_for_response: bool = True,
timeout: float = 2 * 60,
polling_delay: float = 1.0) -> Optional[UUID]:
"""
[ALPHA] Update a particular element of the collection with data validation.
Update a particular element of the collection, doing a deeper check to ensure that
the dependent data objects are still with the (potentially) changed constraints
of this change. This will allow you to make bounds and allowed named/labels changes
to templates.
Parameters
----------
model: ResourceType
The DataConcepts object.
dry_run: bool
Whether to actually update the item or run a dry run of the update operation.
Dry run is intended to be used for validation. Default: false
wait_for_response:
Whether to poll for the eventual response. This changes the return type (see
below).
timeout:
How long to poll for the result before giving up. This is expressed in
(fractional) seconds.
polling_delay:
How long to delay between each polling retry attempt.
Returns
-------
Optional[UUID]
If wait_for_response if True, then this call will poll the backend, waiting
for the eventual job result. In the case of successful validation/update,
a return value of None is provided which indicates success. In the case of
a failure validating or processing the update, an exception (JobFailureError)
is raised and an error message is logged with the underlying reason of the
failure.
If wait_for_response if False, A job ID (of type UUID) is returned that one
can use to poll for the job completion and result with the
:func:`~citrine.resources.DataConceptsCollection.poll_async_update_job`
method.
"""
temp_scope = str(uuid4())
GEMDJson(scope=temp_scope).dumps(
model) # This apparent no-op populates uids
dumped_data = replace_objects_with_links(scrub_none(model.dump()))
recursive_foreach(
model, lambda x: x.uids.pop(temp_scope, None)) # Strip temp uids
scope = CITRINE_SCOPE
id = dumped_data['uids']['id']
if self.dataset_id is None:
raise RuntimeError("Must specify a dataset in order to update "
"a data model object with data validation.")
url = self._get_path() + \
"/" + scope + "/" + id + "/async"
response_json = self.session.put_resource(url,
dumped_data,
params={'dry_run': dry_run})
job_id = response_json["job_id"]
if wait_for_response:
self.poll_async_update_job(job_id,
timeout=timeout,
polling_delay=polling_delay)
# That worked, nothing returned in this case
return None
else:
# TODO: use JobSubmissionResponse here instead
return job_id
