def test_serialize():
"""Serializing a nested object should be identical to individually serializing each piece."""
condition = Condition(name="A condition", value=NominalReal(7, ''))
parameter = Parameter(name="A parameter",
value=NormalReal(mean=17, std=1, units=''))
input_material = MaterialRun(tags="input")
process = ProcessRun(tags="A tag on a process run")
ingredient = IngredientRun(material=input_material, process=process)
material = MaterialRun(tags=["A tag on a material"], process=process)
measurement = MeasurementRun(tags="A tag on a measurement",
conditions=condition,
parameters=parameter,
material=material)
# serialize the root of the tree
native_object = json.loads(dumps(measurement))
# ingredients don't get serialized on the process
assert (len(native_object["context"]) == 5)
assert (native_object["object"]["type"] == LinkByUID.typ)
# serialize all of the nodes
native_batch = json.loads(
dumps([material, process, measurement, ingredient]))
assert (len(native_batch["context"]) == 5)
assert (len(native_batch["object"]) == 4)
assert (all(x["type"] == LinkByUID.typ for x in native_batch["object"]))
def test_material_run():
"""
Test the ability to create a MaterialRun that is linked to a MaterialSpec.
Make sure all enumerated values are respected, and check consistency after
serializing and deserializing.
"""
# Define a property, and make sure that an inappropriate value for origin throws ValueError
with pytest.raises(ValueError):
prop = Property(name="A property", origin="bad origin", value=NominalReal(17, units=''))
# Create a MaterialSpec with a property
prop = Property(name="A property", origin="specified", value=NominalReal(17, units=''))
mat_spec = MaterialSpec(
name="a specification for a material",
properties=PropertyAndConditions(prop),
notes="Funny lookin'"
)
# Make sure that when property is serialized, origin (an enumeration) is serialized as a string
copy_prop = json.loads(dumps(mat_spec))
copy_origin = copy_prop["context"][0]["properties"][0]['property']['origin']
assert isinstance(copy_origin, str)
# Create a MaterialRun, and make sure an inappropriate value for sample_type throws ValueError
with pytest.raises(ValueError):
mat = MaterialRun("name", spec=mat_spec, sample_type="imaginary")
mat = MaterialRun("name", spec=mat_spec, sample_type="virtual")
# ensure that serialization does not change the MaterialRun
copy = loads(dumps(mat))
assert dumps(copy) == dumps(mat), \
"Material run is modified by serialization or deserialization"
def test_sac():
"""Make S&C table and assert that it can be serialized."""
sac = make_strehlow_objects(import_table())
sac_tbl = make_strehlow_table(sac)
# Check that all shapes of records serialize and deserialize
for comp in sac:
assert je.loads(je.dumps(comp)) == comp
# Verify that specs are shared when compounds match
for comp1 in sac:
for comp2 in sac:
# xand
assert (comp1.name == comp2.name) == (comp1.spec.uids == comp2.spec.uids)
# Look at each different combination of Value types in a S&C record
smaller = minimal_subset(sac_tbl['content'])
# Make sure that the diversity of value types isn't lost, e.g. something is being None'd
assert len(smaller) == 162
# Make sure there's no migration with repeated serialization
for row in sac_tbl:
assert je.dumps(je.loads(je.dumps(row))) == je.dumps(row)
# Verify that the serialization trick for mocking a structured table works
json.dumps(json.loads(je.dumps(sac_tbl))["object"], indent=2)
def test_process_id_link():
"""Test that a process run can house a LinkByUID object, and that it survives serde."""
uid = str(uuid4())
proc_link = LinkByUID(scope='id', id=uid)
mat_run = MaterialRun("Another cake", process=proc_link)
copy_material = loads(dumps(mat_run))
assert dumps(copy_material) == dumps(mat_run)
def test_unexpected_serialization():
"""Trying to serialize an unexpected class should throw a TypeError."""
class DummyClass:
def __init__(self, foo):
self.foo = foo
with pytest.raises(TypeError):
dumps(ProcessRun("A process", notes=DummyClass("something")))
def test_attribute_serde():
"""An attribute with a link to an attribute template should be copy-able."""
prop_tmpl = PropertyTemplate(name='prop_tmpl',
bounds=RealBounds(0, 2, 'm'))
prop = Property(name='prop', template=prop_tmpl, value=NominalReal(1, 'm'))
meas_spec = MeasurementSpec("a spec")
meas = MeasurementRun("a measurement", spec=meas_spec, properties=[prop])
assert loads(dumps(prop)) == prop
assert loads(dumps(meas)) == meas
assert isinstance(prop.template, PropertyTemplate)
def test_unexpected_deserialization():
"""Trying to deserialize an unexpected class should throw a TypeError."""
class DummyClass(DictSerializable):
typ = 'dummy_class'
def __init__(self, foo):
self.foo = foo
# DummyClass cannot be serialized since dumps will round-robin serialize
# in the substitute_links method
with pytest.raises(TypeError):
dumps(ProcessRun("A process", notes=DummyClass("something")))
def test_process_spec():
"""Tests that the Process Spec/Run connection persists when serializing."""
# Create the ProcessSpec
condition1 = Condition(name="a condition on the process in general")
spec = ProcessSpec(conditions=condition1)
# Create the ProcessRun with a link to the ProcessSpec from above
condition2 = Condition(name="a condition on this process run in particular")
process = ProcessRun(conditions=condition2, spec=spec)
copy_process = loads(dumps(process))
assert dumps(copy_process.spec) == dumps(spec), \
"Process spec should be preserved through serialization"
def test_process_run():
"""Test that a process run can house a material, and that it survives serde."""
process_run = ProcessRun("Bake a cake", uids={'My_ID': str(17)})
material_run = MaterialRun("A cake", process=process_run)
# Check that a bi-directional link is established
assert material_run.process == process_run
assert process_run.output_material == material_run
copy_material = loads(dumps(material_run))
assert dumps(copy_material) == dumps(material_run)
assert 'output_material' in repr(process_run)
assert 'process' in repr(material_run)
def test_numpy():
"""Test that ndarrays, Series work as well."""
assert len(array_like()) < 5 # In case we extend at some point
if len(array_like()) > 2: # Test numpy
import numpy as np
np_bounds = CategoricalBounds(np.array(["spam", "eggs"], dtype=object))
np_copy = loads(dumps(np_bounds))
assert np_copy == np_bounds
if len(array_like()) > 3: # Test numpy
import pandas as pd
pd_bounds = CategoricalBounds(pd.Series(["spam", "eggs"]))
pd_copy = loads(dumps(pd_bounds))
assert pd_copy == pd_bounds
def test_passthrough_bounds():
"""Test that unspecified Bounds are accepted and set to None."""
template = ProcessTemplate('foo',
conditions=[
(LinkByUID('1', '2'), None),
[LinkByUID('3', '4'), None],
LinkByUID('5', '6'),
ConditionTemplate('foo',
bounds=IntegerBounds(
0, 10)),
])
assert len(template.conditions) == 4
for _, bounds in template.conditions:
assert bounds is None
copied = loads(dumps(template))
assert len(copied.conditions) == 4
for _, bounds in copied.conditions:
assert bounds is None
from_dict = ProcessTemplate.build({
'type':
'process_template',
'name':
'foo',
'conditions': [[
{
'scope': 'foo',
'id': 'bar',
'type': 'link_by_uid',
},
None,
]],
})
assert len(from_dict.conditions) == 1
def test_dict_serialization():
"""Test that a dictionary can be serialized and then deserialized as a gemd object."""
process = ProcessRun("A process")
mat = MaterialRun("A material", process=process)
meas = MeasurementRun("A measurement", material=mat)
copy = loads(dumps(meas.as_dict()))
assert copy == meas
def test_access_data():
"""Demonstrate and test access patterns within the data island."""
binders = {
"Polyethylene Glycol 100M": 0.02,
"Sodium lignosulfonate": 0.004,
"Polyvinyl Acetate": 0.0001
}
powders = {"Al2O3": 0.96}
island = make_data_island(density=1.0,
bulk_modulus=300.0,
firing_temperature=750.0,
binders=binders,
powders=powders,
tag="Me")
# read the density value
assert (island.measurements[0].properties[0].value == NominalReal(1.0, ''))
# read the bulk modulus value
assert (island.measurements[0].properties[1].value == NormalReal(
300.0, 3.0, ''))
# read the firing temperature
assert (island.process.conditions[0].value == UniformReal(
749.5, 750.5, 'degC'))
assert (island.process.parameters[0].value == DiscreteCategorical(
{"hot": 1.0}))
# read the quantity of alumina
quantities = island.process.ingredients[0].material.process.conditions[
0].value.quantities
assert (list(keyfilter(lambda x: x == "Al2O3",
quantities).values())[0] == 0.96)
# check that the serialization results in the correct number of objects in the preface
# (note that neither measurements nor ingredients are serialized)
assert (len(json.loads(dumps(island))["context"]) == 26)
def test_material_soft_link():
"""Test that a measurement run can link to a material run, and that it survives serde."""
dye = MaterialRun("rhodamine",
file_links=FileLink(filename='a.csv', url='/a/path'))
assert dye.measurements == [], "default value of .measurements should be an empty list"
# The .measurements member should not be settable
with pytest.raises(AttributeError):
dye.measurements = [MeasurementRun("Dummy")]
absorbance = MeasurementRun(name="Absorbance",
uids={'id': str(uuid4())},
properties=[
Property(name='Abs at 500 nm',
value=NominalReal(0.1, ''))
])
assert absorbance.material is None, "Measurements should have None as the material by default"
absorbance.material = dye
assert absorbance.material == dye, "Material not set correctly for measurement"
assert dye.measurements == [
absorbance
], "Soft-link from material to measurement not created"
fluorescence = MeasurementRun(name="Fluorescence",
uids={'id': str(uuid4())},
properties=[
Property(name='PL counts at 550 nm',
value=NominalReal(30000, ''))
],
material=dye)
assert fluorescence.material == dye, "Material not set correctly for measurement"
assert dye.measurements == [absorbance, fluorescence], \
"Soft-link from material to measurements not created"
assert loads(dumps(absorbance)) == absorbance, \
"Measurement should remain unchanged when serialized"
assert loads(dumps(fluorescence)) == fluorescence, \
"Measurement should remain unchanged when serialized"
assert 'measurements' in repr(dye)
assert 'material' in repr(fluorescence)
assert 'material' in repr(absorbance)
subbed = substitute_links(dye)
assert 'measurements' in repr(subbed)
def test_uid_deser():
"""Test that uids continue to be a CaseInsensitiveDict after deserialization."""
material = MaterialRun("Input material", tags="input", uids={'Sample ID': '500-B'})
ingredient = IngredientRun(material=material)
ingredient_copy = loads(dumps(ingredient))
assert isinstance(ingredient_copy.uids, CaseInsensitiveDict)
assert ingredient_copy.material == material
assert ingredient_copy.material.uids['sample id'] == material.uids['Sample ID']
def test_measurement_spec():
"""Test the measurement spec/run connection survives ser/de."""
condition = Condition(name="Temp condition",
value=NominalReal(nominal=298, units='kelvin'))
parameter = Parameter(name="Important parameter")
spec = MeasurementSpec(name="Precise way to do a measurement",
parameters=parameter,
conditions=condition)
# Create a measurement run from this measurement spec
measurement = MeasurementRun("The Measurement",
conditions=condition,
spec=spec)
copy = loads(dumps(measurement))
assert dumps(copy.spec) == dumps(measurement.spec), \
"Measurement spec should be preserved if measurement run is serialized"
def test_recursive_equals():
"""Verify that the recursive/crawling equals behaves well."""
cake = make_cake()
copy = loads(dumps(cake))
assert cake == copy
copy.process.ingredients[0].material.process.ingredients[0].material.tags.append('Hi')
assert cake != copy
def test_json_serde():
"""Test that values can be ser/de using our custom json loads/dumps."""
# Enums are only used in the context of another class --
# it is not possible to deserialize to enum with the current
# serialization strategy (plain string) without this context.
original = Property(name="foo", origin=Origin.MEASURED)
copy = loads(dumps(original))
assert original == copy
def test_source():
"""Test that source can be set, serialized, and deserialized."""
source = PerformedSource(performed_by="Marie Curie",
performed_date="1898-07-01")
measurement = MeasurementRun(name="Polonium", source=source)
assert loads(dumps(measurement)).source.performed_by == "Marie Curie"
with pytest.raises(TypeError):
MeasurementRun(name="Polonium", source="Marie Curie on 1898-07-01")
def test_link_by_uid():
"""Test that linking works."""
root = MaterialRun(name='root', process=ProcessRun(name='root proc'))
leaf = MaterialRun(name='leaf', process=ProcessRun(name='leaf proc'))
IngredientRun(process=root.process, material=leaf)
IngredientRun(process=root.process, material=LinkByUID.from_entity(leaf))
copy = loads(dumps(root))
assert copy.process.ingredients[0].material == copy.process.ingredients[
1].material
def test_condition_template():
"""Test creation and serde of condition templates."""
bounds = RealBounds(2.5, 10.0, default_units='cm')
template = ConditionTemplate("Chamber width", bounds=bounds, description="width of chamber")
assert template.uids is not None # uids should be added automatically
# Take template through a serde cycle and ensure that it is unchanged
assert ConditionTemplate.build(template.dump()) == template
# A more complicated cycle that goes through both gemd-python and citrine-python serde.
assert ConditionTemplate.build(loads(dumps(template.dump())).as_dict()) == template
def test_ingredient_spec():
"""Tests that a process can house an ingredient, and that pairing survives serialization."""
# Create a ProcessSpec
proc_spec = ProcessSpec(name="a process spec", tags=["tag1", "tag2"])
IngredientSpec(name='Input',
material=MaterialSpec(name='Raw'),
process=proc_spec)
# Make copies of both specs
proc_spec_copy = loads(dumps(proc_spec))
assert proc_spec_copy == proc_spec, "Full structure wasn't preserved across serialization"
def test_material_spec():
"""Test that Process/Material Spec link survives serialization."""
# Create a ProcessSpec
proc_spec = ProcessSpec(name="a process spec", tags=["tag1", "tag2"])
# Create MaterialSpec without a ProcessSpec
prop = Property(name="The material is a solid",
value=DiscreteCategorical(probabilities="solid"))
mat_spec = MaterialSpec(name="a material spec",
properties=PropertyAndConditions(prop))
assert mat_spec.process is None, \
"MaterialSpec should be initialized with no ProcessSpec, by default"
# Assign a ProcessSpec to mat_spec, first ensuring that the type is enforced
with pytest.raises(TypeError):
mat_spec.process = 17
mat_spec.process = proc_spec
# Assert circular links
assert dumps(proc_spec.output_material.process) == dumps(proc_spec), \
"ProcessSpec should link to MaterialSpec that links back to itself"
assert dumps(mat_spec.process.output_material) == dumps(mat_spec), \
"MaterialSpec should link to ProcessSpec that links back to itself"
# Make copies of both specs
mat_spec_copy = loads(dumps(mat_spec))
proc_spec_copy = loads(dumps(proc_spec))
assert proc_spec_copy.output_material == mat_spec, \
"Serialization should preserve link from ProcessSpec to MaterialSpec"
assert mat_spec_copy.process == proc_spec, \
"Serialization should preserve link from MaterialSpec to ProcessSpec"
def test_ingredient_run():
"""Tests that a process can house an ingredient, and that pairing survives serialization."""
# Create a ProcessSpec
proc_run = ProcessRun(name="a process spec", tags=["tag1", "tag2"])
ingred_run = IngredientRun(material=MaterialRun(name='Raw'), process=proc_run)
# Make copies of both specs
proc_run_copy = loads(dumps(proc_run))
assert proc_run_copy == proc_run, "Full structure wasn't preserved across serialization"
assert 'process' in repr(ingred_run)
assert 'ingredients' in repr(proc_run)
def test_many_ingredients():
"""Test that ingredients remain connected to processes when round-robined through json."""
proc = ProcessRun("foo", spec=ProcessSpec("sfoo"))
expected = []
for i in range(10):
mat = MaterialRun(name=str(i), spec=MaterialSpec("s{}".format(i)))
i_spec = IngredientSpec(name="i{}".format(i), material=mat.spec, process=proc.spec)
IngredientRun(process=proc, material=mat, spec=i_spec)
expected.append("i{}".format(i))
reloaded = loads(dumps(proc))
assert len(list(reloaded.ingredients)) == 10
names = [x.name for x in reloaded.ingredients]
assert sorted(names) == sorted(expected)
def test_measurement_example():
"""Simple driver to populate flex_measurements.json and validate that it has contents."""
num_measurements = 4
results = make_demo_measurements(num_measurements, extra_tags={"demo"})
with open("flex_measurements.json", "w") as f:
f.write(dumps(results, indent=2))
with open("flex_measurements.json", "r") as f:
copy = load(f)
assert len(copy) == len(results)
assert all("demo" in x.tags for x in copy)
assert all("my_id" in x.uids for x in copy)
def test_process_attachment():
"""Test that a process can be attached to a material, and that the connection survives serde"""
cake = MaterialRun('Final cake')
cake.process = ProcessRun('Icing', uids={'id': '12345'})
cake_data = cake.dump()
cake_copy = loads(dumps(cake_data)).as_dict()
assert cake_copy['name'] == cake.name
assert cake_copy['uids'] == cake.uids
assert cake.process.uids['id'] == cake_copy['process'].uids['id']
reconstituted_cake = MaterialRun.build(cake_copy)
assert isinstance(reconstituted_cake, MaterialRun)
assert isinstance(reconstituted_cake.process, ProcessRun)
def test_cake_sigs():
"""Verify that all arguments for create methods work as expected."""
templates = make_cake_templates()
tmpl_snap = dumps(templates)
specs = make_cake_spec(templates)
spec_snap = dumps(specs)
assert dumps(templates) == tmpl_snap
cake1 = make_cake(seed=27, cake_spec=specs, tmpl=templates)
assert dumps(templates) == tmpl_snap
assert dumps(specs) == spec_snap
filelink = FileLink(filename='The name of the file', url='www.file.gov')
cake2 = make_cake(seed=27, cake_spec=specs, tmpl=templates, toothpick_img=filelink)
assert filelink.filename not in dumps(cake1)
assert filelink.filename in dumps(cake2)
assert cake1.uids == cake2.uids
def test_link_by_uid():
"""Test that linking works."""
root = MaterialRun(name='root', process=ProcessRun(name='root proc'))
leaf = MaterialRun(name='leaf', process=ProcessRun(name='leaf proc'))
IngredientRun(process=root.process, material=leaf)
IngredientRun(process=root.process,
material=LinkByUID.from_entity(leaf, scope='id'))
# Paranoid assertions about equality's symmetry since it's implemented in 2 places
assert root.process.ingredients[0].material == root.process.ingredients[
1].material
assert root.process.ingredients[0].material.__eq__(
root.process.ingredients[1].material)
assert root.process.ingredients[1].material.__eq__(
root.process.ingredients[0].material)
# Verify hash collision on equal LinkByUIDs
assert LinkByUID.from_entity(leaf) in {LinkByUID.from_entity(leaf)}
copy = loads(dumps(root))
assert copy.process.ingredients[0].material == copy.process.ingredients[
1].material
template_scope = args[0]
else:
template_scope = DEMO_TEMPLATE_SCOPE
imported_table = import_table(FULL_TABLE)
full_compounds = make_strehlow_objects(imported_table, template_scope)
full_table = make_strehlow_table(full_compounds)
small_table = minimal_subset(full_table['content'])
todo = set(_fingerprint(x) for x in small_table)
print('Total number of prototypes: {}'.format(len(small_table)))
reduced_list = []
for (raw, clean) in zip(imported_table, full_table['content']):
fp = _fingerprint(clean)
if fp in todo:
reduced_list.append(raw)
todo.remove(fp)
if not todo:
break
with open(os.path.join(os.path.dirname(__file__), SMALL_TABLE), 'w') as f:
json.dump(reduced_list, f, indent=2)
print("\n\nJSON -- Training table")
import gemd.json as je
print(json.dumps(json.loads(je.dumps(full_table))["object"], indent=2))
print("\n\nCSV -- Display table")
display = make_display_table(full_table)
for row in display:
print(','.join(map(lambda x: str(x), row)))
