def test_unit_handling(self):
"""
Tests unit handling with a simple model that calculates the area of a rectangle as the
product of two lengths.
In this case the input lengths are provided in centimeters and meters.
Tests whether the input units are properly coerced into canonical types.
Tests whether the output units are properly set.
Tests whether the model returns as predicted.
Returns:
None
"""
L = Symbol('l', ['L'], ['L'], units=[1.0, [['centimeter', 1.0]]], shape=[1])
A = Symbol('a', ['A'], ['A'], units=[1.0, [['centimeter', 2.0]]], shape=[1])
get_area_config = {
'name': 'area',
# 'connections': [{'inputs': ['l1', 'l2'], 'outputs': ['a']}],
'equations': ['a = l1 * l2'],
# 'unit_map': {'l1': "cm", "l2": "cm", 'a': "cm^2"}
'symbol_property_map': {"a": A, "l1": L, "l2": L}
}
model = EquationModel(**get_area_config)
out = model.evaluate({'l1': QuantityFactory.create_quantity(L, 1, 'meter'),
'l2': QuantityFactory.create_quantity(L, 2)}, allow_failure=False)
self.assertTrue(math.isclose(out['a'].magnitude, 200.0))
self.assertTrue(out['a'].units == A.units)
def test_model_returns_complex(self):
# This tests model failure with scalar complex.
# Quantity class has other more thorough tests.
A = Symbol('a', ['A'], ['A'], units='dimensionless', shape=1)
B = Symbol('b', ['B'], ['B'], units='dimensionless', shape=1)
for sym in (B, A):
Registry("symbols")[sym] = sym
Registry("units")[sym] = sym.units
get_config = {
'name': 'add_complex_value',
# 'connections': [{'inputs': ['b'], 'outputs': ['a']}],
'equations': ['a = b + 1j'],
# 'unit_map': {'a': "dimensionless", 'a': "dimensionless"}
'variable_symbol_map': {
"a": A,
"b": B
}
}
model = EquationModel(**get_config)
out = model.evaluate({'b': QuantityFactory.create_quantity(B, 5)},
allow_failure=True)
self.assertFalse(out['successful'])
self.assertEqual(out['message'],
'Evaluation returned invalid values (complex)')
out = model.evaluate({'b': QuantityFactory.create_quantity(B, 5j)},
allow_failure=True)
self.assertTrue(out['successful'])
self.assertTrue(np.isclose(out['a'].magnitude, 6j))
def test(self, inputs, outputs):
"""
Runs a test of the model to determine whether its operation
is consistent with the specified inputs and outputs
Args:
inputs (dict): set of input names to values
outputs (dict): set of output names to values
Returns (bool): True if test succeeds
"""
evaluate_inputs = self.map_symbols_to_properties(inputs)
unit_map_as_properties = self.map_symbols_to_properties(self.unit_map)
evaluate_inputs = {
s: QuantityFactory.create_quantity(s, v,
unit_map_as_properties.get(s))
for s, v in evaluate_inputs.items()}
evaluate_outputs = self.evaluate(evaluate_inputs, allow_failure=False)
evaluate_outputs = self.map_properties_to_symbols(evaluate_outputs)
errmsg = "{} model test failed on ".format(self.name) + "{}\n"
errmsg += "{}(test data) = {}\n"#.format(k, known_output)
errmsg += "{}(model output) = {}"#.format(k, plug_in_output)
for k, known_output in outputs.items():
symbol = self.symbol_property_map[k]
units = self.unit_map.get(k)
known_quantity = QuantityFactory.create_quantity(symbol, known_output, units)
evaluate_output = evaluate_outputs[k]
if not known_quantity.has_eq_value_to(evaluate_output):
errmsg = errmsg.format("evaluate", k, evaluate_output,
k, known_quantity)
raise ModelEvaluationError(errmsg)
return True
def test_add_default_quantities(self):
material = Material(add_default_quantities=True)
self.assertEqual(list(material['temperature'])[0],
QuantityFactory.create_quantity("temperature", 300,
provenance=ProvenanceElement(model='default')))
self.assertEqual(list(material['relative_permeability'])[0],
QuantityFactory.create_quantity("relative_permeability", 1,
provenance=ProvenanceElement(model='default')))
def setUp(self):
# Create some test properties and a few base objects
self.q1 = QuantityFactory.create_quantity(DEFAULT_SYMBOLS['bulk_modulus'],
ureg.Quantity.from_tuple([200, [['gigapascals', 1]]]))
self.q2 = QuantityFactory.create_quantity(DEFAULT_SYMBOLS['shear_modulus'],
ureg.Quantity.from_tuple([100, [['gigapascals', 1]]]))
self.q3 = QuantityFactory.create_quantity(DEFAULT_SYMBOLS['bulk_modulus'],
ureg.Quantity.from_tuple([300, [['gigapascals', 1]]]))
self.material = Material()
self.graph = Graph()
def test_get_weight(self):
q1 = QuantityFactory.create_quantity("band_gap", 3.2)
wt = get_weight(q1)
self.assertEqual(wt, 1)
p2 = ProvenanceElement(model="model_2", inputs=[q1])
q2 = QuantityFactory.create_quantity("refractive_index", 4, provenance=p2)
wt2 = get_weight(q2, {"model_2": 0.5})
self.assertEqual(wt2, 0.5)
p3 = ProvenanceElement(model="model_3", inputs=[q2])
q3 = QuantityFactory.create_quantity("bulk_modulus", 100, provenance=p3)
wt3 = get_weight(q3, {"model_3": 0.25, "model_2": 0.5})
self.assertEqual(wt3, 0.125)
def setUpClass(cls):
add_builtin_symbols_to_registry()
# Create some test properties and a few base objects
cls.q1 = QuantityFactory.create_quantity(
Registry("symbols")['bulk_modulus'],
ureg.Quantity.from_tuple([200, [['gigapascals', 1]]]))
cls.q2 = QuantityFactory.create_quantity(
Registry("symbols")['shear_modulus'],
ureg.Quantity.from_tuple([100, [['gigapascals', 1]]]))
cls.q3 = QuantityFactory.create_quantity(
Registry("symbols")['bulk_modulus'],
ureg.Quantity.from_tuple([300, [['gigapascals', 1]]]))
cls.material = None
cls.graph = Graph()
def get_materials_for_mpids(self, mpids, filter_null_properties=True):
"""
Retrieve a list of Materials from the materials
Project for a given list of Materials Project IDs.
Args:
mpids: a list of Materials Project IDs
Returns:
"""
materials_properties = self.get_properties_for_mpids(
mpids, filter_null_properties=filter_null_properties)
materials = []
for material_properties in materials_properties:
material = Material()
for property_name, property_value in material_properties.items():
provenance = ProvenanceElement(source='Materials Project')
quantity = QuantityFactory.create_quantity(
self.mapping[property_name],
property_value,
provenance=provenance)
material.add_quantity(quantity)
materials.append(material)
return materials
def evaluate(input_rows, data, aggregate):
quantities = [
QuantityFactory.create_quantity(
symbol_type=ROW_IDX_TO_SYMBOL_NAME[idx],
value=ureg.parse_expression(row['Editable Value']))
for idx, row in enumerate(input_rows) if row['Editable Value']
]
if data and len(data) > 0:
quantities += json.loads(data, cls=MontyDecoder).values()
if not quantities:
raise PreventUpdate
material = Material()
for quantity in quantities:
material.add_quantity(quantity)
graph = Graph()
output_material = graph.evaluate(material)
if aggregate:
output_quantities = output_material.get_aggregated_quantities(
).values()
else:
output_quantities = output_material.get_quantities()
output_rows = [{
'Property': quantity.symbol.display_names[0],
'Value': quantity.pretty_string(sigfigs=3)
} for quantity in output_quantities]
output_table = dt.DataTable(id='output-table',
rows=output_rows,
editable=False)
# TODO: clean up
input_quantity_names = [q.symbol.name for q in quantities]
derived_quantity_names = set(
[q.symbol.name for q in output_quantities]) - \
set(input_quantity_names)
material_graph_data = graph_conversion(
graph.get_networkx_graph(),
nodes_to_highlight_green=input_quantity_names,
nodes_to_highlight_yellow=list(derived_quantity_names))
options = AESTHETICS['global_options']
options['edges']['color'] = '#000000'
output_graph = html.Div(GraphComponent(id='material-graph',
graph=material_graph_data,
options=options),
style={
'width': '100%',
'height': '400px'
})
return [output_graph, html.Br(), output_table]
def test_get_sse(self):
mats = [Material([QuantityFactory.create_quantity("band_gap", n)]) for n in range(1, 5)]
benchmarks = [{"band_gap": 1.1*n} for n in range(1, 5)]
err = get_sse(mats, benchmarks)
test_val = sum([0.01*n**2 for n in range(1, 5)])
self.assertAlmostEqual(err, test_val)
# Big dataset
err = get_sse(self.evaluated, self.benchmarks)
self.assertAlmostEqual(err, 173.5710251)
def setUp(self):
path = os.path.join(TEST_DIR, "fitting_test_data.csv")
test_data = pd.read_csv(path)
graph = Graph()
materials = [Material([QuantityFactory.create_quantity("band_gap", bg)])
for bg in test_data['band_gap']]
self.evaluated = [graph.evaluate(mat) for mat in materials]
self.benchmarks = [{"refractive_index": n}
for n in test_data['refractive_index']]
def setUpClass(cls):
add_builtin_models_to_registry()
# This is the visible light dataset used in the propnet paper
path = os.path.join(TEST_DATA_DIR, "vis_bg_ri_data.csv")
test_data = pd.read_csv(path)
graph = Graph()
materials = [
Material([QuantityFactory.create_quantity("band_gap", bg)])
for bg in test_data['Band Gap']
]
cls.evaluated = [graph.evaluate(mat) for mat in materials]
cls.benchmarks = [{
"refractive_index": n
} for n in test_data['Refractive Index']]
def to_quantity(self, lookup=None):
"""
Converts the current StorageQuantity to the appropriate BaseQuantity-derived
object.
Verifies that the object's provenance has complete values, which would be
missing if read in from a JSON-serialized dictionary. If values are found
to be missing, a lookup function/dictionary is required to reconstitute the data.
Lookup dictionaries should be keyed by internal ID, and have values which are
dictionaries with the fields shown below. Lookup functions should take one argument,
internal ID, and return a dictionary with the fields below.
Lookup return value construction:
value: (id) value of the quantity
units: (str) units of quantity, None if no units
uncertainty: (int, float, complex) uncertainty value, None if no uncertainty
Args:
lookup: (dict or function) lookup container for missing values. Not required
if self.needs_lookup() is False
Returns: (BaseQuantity) converted BaseQuantity-derived object
"""
if self.needs_lookup() and not lookup:
raise ValueError(
"StorageQuantity cannot be converted to BaseQuantity-derived"
" object because it is missing values in provenance inputs. "
"Please provide a lookup dictionary or function with the "
"following keys: {}".format(self.get_missing_keys()))
provenance_in = \
self._provenance.to_provenance_element(lookup=lookup) \
if self._provenance else None
out = QuantityFactory.create_quantity(symbol_type=self._symbol_type,
value=self._value,
units=self._units,
tags=self._tags,
provenance=provenance_in,
uncertainty=self._uncertainty)
out._internal_id = self._internal_id
return out
def aggregate_quantities(quantities, model_score_dict=None):
"""
Simple method for aggregating a set of quantities
Args:
quantities:
model_score_dict:
Returns:
"""
symbol = next(iter(quantities)).symbol
if not all([q.symbol == symbol for q in quantities]):
raise ValueError("Quantities passed to aggregate must be same symbol")
weights = [get_weight(q, model_score_dict) for q in quantities]
result_value = sum(
[w * q.value for w, q in zip(weights, quantities)]) / sum(weights)
return QuantityFactory.create_quantity(symbol, result_value)
def test_model_returns_nan(self):
# This tests model failure with scalar nan.
# Quantity class has other more thorough tests.
A = Symbol('a', ['A'], ['A'], units='dimensionless', shape=1)
B = Symbol('b', ['B'], ['B'], units='dimensionless', shape=1)
get_config = {
'name': 'equality',
# 'connections': [{'inputs': ['b'], 'outputs': ['a']}],
'equations': ['a = b'],
# 'unit_map': {'a': "dimensionless", 'a': "dimensionless"}
'symbol_property_map': {"a": A, "b": B}
}
model = EquationModel(**get_config)
out = model.evaluate({'b': QuantityFactory.create_quantity(B, float('nan'))},
allow_failure=True)
self.assertFalse(out['successful'])
self.assertEqual(out['message'], 'Evaluation returned invalid values (NaN)')
def get_materials_for_mpids(self, mpids, filter_null_values=True):
"""
Retrieve a list of Materials from the materials
Project for a given list of Materials Project IDs.
Args:
mpids: a list of Materials Project IDs
Returns:
"""
materials_quantities = self.get_quantities_for_mpids(
mpids,
filter_null_values=filter_null_values,
include_date_created=True)
materials = []
for material_quantities in materials_quantities:
material = Material()
try:
date_created = material_quantities.pop('created_at')
except KeyError:
date_created = None
for symbol_name, value in material_quantities.items():
provenance = ProvenanceElement(
source={
'source': 'Materials Project',
'source_key': material_quantities.get(
'material_id', None),
'date_created': date_created
})
quantity = QuantityFactory.create_quantity(
self.mapping[symbol_name],
value,
units=Registry("units").get(self.mapping[symbol_name],
None),
provenance=provenance)
material.add_quantity(quantity)
materials.append(material)
return materials
def transform_properties_to_material(self, material_data):
"""
Produces a propnet Material object from a dictionary of AFLOW materials data.
Args:
material_data (dict): AFLOW materials data, keyed by AFLOW keyword, as
Python native types, not as strings as they are stored in AFLOW.
Returns:
propnet.core.materials.Material: propnet material containing the AFLOW data
"""
qs = []
auid = material_data.get('auid')
date_created = material_data.get('aflowlib_date')
if date_created:
date, tz = date_created.rsplit("GMT", 1)
tz = "GMT{:+05d}".format(int(tz) * 100)
date_object = datetime.strptime(date + tz, "%Y%m%d_%H:%M:%S_%Z%z")
date_created = date_object.strftime("%Y-%m-%d %H:%M:%S")
for prop, value in material_data.items():
if value is not None and prop in self.mapping:
provenance = ProvenanceElement(
source={'source': 'AFLOW',
'source_key': auid,
'date_created': date_created}
)
if prop in self.transform_func:
value = self.transform_func[prop](value)
if value is None:
continue
q = QuantityFactory.create_quantity(
self.mapping.get(prop),
value,
units=self.unit_map.get(prop), provenance=provenance
)
qs.append(q)
return Material(qs)
def process(self, item):
if self.graph_parallel and not self.allow_child_process and \
current_process().name != "MainProcess":
logger.warning(
"It appears derive_quantities() is running "
"in a child process, possibly in a parallelized "
"Runner.\nThis is not recommended and will deteriorate "
"performance.")
# Define quantities corresponding to materials doc fields
# Attach quantities to materials
item = MontyDecoder().process_decoded(item)
logger.info("Populating material for %s", item['task_id'])
material = Material()
if 'created_at' in item.keys():
date_created = item['created_at']
else:
date_created = None
provenance = ProvenanceElement(
source={
"source": self.source_name,
"source_key": item['task_id'],
"date_created": date_created
})
for mkey, property_name in self.materials_symbol_map.items():
value = pydash.get(item, mkey)
if value:
material.add_quantity(
QuantityFactory.create_quantity(
property_name,
value,
units=Registry("units").get(property_name, None),
provenance=provenance))
# Add custom things, e. g. computed entry
computed_entry = get_entry(item)
if computed_entry:
material.add_quantity(
QuantityFactory.create_quantity("computed_entry",
computed_entry,
provenance=provenance))
else:
logger.info("Unable to create computed entry for {}".format(
item['task_id']))
material.add_quantity(
QuantityFactory.create_quantity("external_identifier_mp",
item['task_id'],
provenance=provenance))
input_quantities = material.symbol_quantities_dict
# Use graph to generate expanded quantity pool
logger.info("Evaluating graph for %s", item['task_id'])
new_material = self._graph_evaluator.evaluate(
material, timeout=self.graph_timeout)
# Format document and return
logger.info("Creating doc for %s", item['task_id'])
# Gives the initial inputs that were used to derive properties of a
# certain material.
doc = {
"inputs": [
StorageQuantity.from_quantity(q)
for q in chain.from_iterable(input_quantities.values())
]
}
for symbol, quantities in new_material.symbol_quantities_dict.items():
# If no new quantities of a given symbol were derived (i.e. if the initial
# input quantity/ies is/are the only one/s listed in the new material) then don't add
# that quantity to the propnet entry document as a derived quantity.
if len(quantities) == len(input_quantities[symbol]):
continue
sub_doc = {}
try:
# Write out all quantities as dicts including the
# internal ID for provenance tracing
qs = [
jsanitize(StorageQuantity.from_quantity(q), strict=True)
for q in quantities
]
except AttributeError as ex:
# Check to see if this is an error caused by an object
# that is not JSON serializable
msg = ex.args[0]
if "object has no attribute 'as_dict'" in msg:
# Write error to db and logger
errmsg = "Quantity of Symbol '{}' is not ".format(symbol.name) + \
"JSON serializable. Cannot write quantities to database!"
logger.error(errmsg)
sub_doc['error'] = errmsg
qs = []
else:
# If not, re-raise the error
raise ex
sub_doc['quantities'] = qs
doc[symbol.name] = sub_doc
aggregated_quantities = new_material.get_aggregated_quantities()
for symbol, quantity in aggregated_quantities.items():
if symbol.name not in doc:
# No new quantities were derived
continue
# Store mean and std dev for aggregated quantities
sub_doc = {
"mean": unumpy.nominal_values(quantity.value).tolist(),
"std_dev": unumpy.std_devs(quantity.value).tolist(),
"units":
quantity.units.format_babel() if quantity.units else None,
"title": quantity.symbol.display_names[0]
}
# Symbol Name -> Sub_Document, listing all Quantities of that type.
doc[symbol.name].update(sub_doc)
doc.update({
"task_id": item["task_id"],
"pretty_formula": item.get("pretty_formula"),
"deprecated": item.get("deprecated", False)
})
if self.include_sandboxed:
doc.update({'sbxn': item.get("sbxn", [])})
return jsanitize(doc, strict=True)
def evaluate(input_rows, data, aggregate):
quantities = []
for idx, row in enumerate(input_rows):
if row['Editable Value']:
try:
value = ureg.parse_expression(row['Editable Value'])
units = Registry("units").get(ROW_IDX_TO_SYMBOL_NAME[idx])
value.ito(units)
except Exception:
# Someone put an invalid value in the table
# TODO: Make error known to the user
raise PreventUpdate
q = QuantityFactory.create_quantity(
symbol_type=ROW_IDX_TO_SYMBOL_NAME[idx], value=value)
quantities.append(q)
if data and len(data) > 0:
quantities += json.loads(data, cls=MontyDecoder).values()
if not quantities:
raise PreventUpdate
material = Material()
for quantity in quantities:
material.add_quantity(quantity)
output_material = graph_evaluator.evaluate(material, timeout=5)
if aggregate:
aggregated_quantities = output_material.get_aggregated_quantities()
non_aggregatable_quantities = [
v for v in output_material.get_quantities()
if v.symbol not in aggregated_quantities
]
output_quantities = list(
aggregated_quantities.values()) + non_aggregatable_quantities
else:
output_quantities = output_material.get_quantities()
output_rows = [{
'Property': quantity.symbol.display_names[0],
'Value': quantity.pretty_string(sigfigs=3)
} for quantity in output_quantities]
output_table = dt.DataTable(id='output-table',
data=output_rows,
columns=[{
'id': val,
'name': val
} for val in ('Property', 'Value')],
editable=False,
**DATA_TABLE_STYLE)
# TODO: clean up
input_quantity_names = [q.symbol for q in quantities]
derived_quantity_names = \
set([q.symbol for q in output_quantities]) - \
set(input_quantity_names)
models_evaluated = set(
output_q.provenance.model
for output_q in output_material.get_quantities())
models_evaluated = [
Registry("models").get(m) for m in models_evaluated
if Registry("models").get(m) is not None
]
material_graph_data = graph_conversion(
propnet_nx_graph,
derivation_pathway={
'inputs': input_quantity_names,
'outputs': list(derived_quantity_names),
'models': models_evaluated
})
output_graph = html.Div(children=[
dcc.Checklist(id='material-graph-options',
options=[{
'label': 'Show models',
'value': 'show_models'
}, {
'label': 'Show properties',
'value': 'show_properties'
}],
value=['show_properties'],
labelStyle={'display': 'inline-block'}),
Cytoscape(id='material-graph',
elements=material_graph_data,
stylesheet=GRAPH_STYLESHEET,
layout=GRAPH_LAYOUT_CONFIG,
**GRAPH_SETTINGS['full_view'])
])
return [output_graph, html.Br(), output_table]
def setUp(self):
# Inspiration was taken from the GraphTest class
# I tried to construct the dictionaries for comparison
# without writing out every one explicity by reusing
# information where it was applicable.
# If this is too unreadable, can change to writing it
# out explicity in a JSON file and importing it. Would
# still need to replace some fields dynamically.
symbols = StorageTest.generate_symbols()
self.custom_syms_as_dicts = {
k: {'@module': 'propnet.core.symbols',
'@class': 'Symbol',
'name': k,
'display_names': [k],
'display_symbols': [k],
'units': (1, ()),
'shape': 1,
'object_type': None,
'comment': None,
'category': 'property',
'constraint': None,
'default_value': None} for k in ['A', 'B', 'C']
}
self.custom_syms_as_dicts['C'].update(
{"units": None,
"shape": None,
"object_type": "str",
"category": "object"})
self.custom_symbols_json = copy.deepcopy(self.custom_syms_as_dicts)
for k in ['A', 'B']:
self.custom_symbols_json[k]['units'] = [1, []]
a = [QuantityFactory.create_quantity(symbols['A'], 19),
QuantityFactory.create_quantity(symbols['A'], 23)]
b = [QuantityFactory.create_quantity(symbols['B'], 38,
provenance=ProvenanceElement(model='model1',
inputs=[a[0]])),
QuantityFactory.create_quantity(symbols['B'], 46,
provenance=ProvenanceElement(model='model1',
inputs=[a[1]]))]
self.quantities_custom_symbol = {"A": a,
"B": b}
self.sq_custom_sym_as_dicts = {
k: [{'@module': 'propnet.dbtools.storage',
'@class': 'StorageQuantity',
'internal_id': vv._internal_id,
'data_type': 'NumQuantity',
'symbol_type': symbols[k],
'value': vv.magnitude,
'units': 'dimensionless',
'provenance': ProvenanceStore.from_provenance_element(vv.provenance),
'tags': [],
'uncertainty': None} for vv in v] for k, v in self.quantities_custom_symbol.items()
}
provenances_json = {
"A": [{'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStore',
'model': None,
'inputs': None,
'source': aa.provenance.source} for aa in a]}
provenances_json['B'] = [
{'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStore',
'model': 'model1',
'inputs': [{'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStoreQuantity',
'data_type': 'NumQuantity',
'symbol_type': self.custom_symbols_json['A'],
'internal_id': q.provenance.inputs[0]._internal_id,
'tags': [],
'provenance': p}],
'source': q.provenance.source} for q, p in zip(b, provenances_json['A'])]
self.sq_custom_sym_json = copy.deepcopy(self.sq_custom_sym_as_dicts)
for sym in ['A', 'B']:
for q, p in zip(self.sq_custom_sym_json[sym], provenances_json[sym]):
q['symbol_type'] = self.custom_symbols_json[sym]
q['provenance'] = p
band_gaps = [QuantityFactory.create_quantity('band_gap', 3.3, 'eV'),
QuantityFactory.create_quantity('band_gap', 2.1, 'eV')]
bg_ri_model = DEFAULT_MODEL_DICT['band_gap_refractive_index_moss']
refractive_indices = [bg_ri_model.evaluate({"Eg": bg}).pop('refractive_index') for bg in band_gaps]
self.quantities_canonical_symbol = {"band_gaps": band_gaps,
"refractive_indices": refractive_indices}
self.sq_canonical_sym_as_dicts_no_value = copy.deepcopy(self.sq_custom_sym_as_dicts)
self.sq_canonical_sym_as_dicts_no_value['band_gaps'] = self.sq_canonical_sym_as_dicts_no_value.pop('A')
self.sq_canonical_sym_as_dicts_no_value['refractive_indices'] = self.sq_canonical_sym_as_dicts_no_value.pop('B')
for d, sq in zip(self.sq_canonical_sym_as_dicts_no_value['band_gaps'], band_gaps):
d.update({
"internal_id": sq._internal_id,
"symbol_type": "band_gap",
"units": "electron_volt",
"provenance": ProvenanceStore.from_provenance_element(sq.provenance)
})
d.pop('value')
for d, sq in zip(self.sq_canonical_sym_as_dicts_no_value['refractive_indices'], refractive_indices):
d.update({
"internal_id": sq._internal_id,
"symbol_type": "refractive_index",
"units": "dimensionless",
"provenance": ProvenanceStore.from_provenance_element(sq.provenance)
})
d.pop('value')
self.sq_canonical_sym_values = {"band_gaps": [3.3, 2.1],
"refractive_indices": [2.316340583741216, 2.593439239956374]}
provenances_json['band_gaps'] = [
{'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStore',
'model': None,
'inputs': None,
'source': bg.provenance.source}
for bg in band_gaps
]
provenances_json['refractive_indices'] = [{
'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStore',
'model': 'band_gap_refractive_index_moss',
'inputs': [{'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStoreQuantity',
'data_type': 'NumQuantity',
'symbol_type': 'band_gap',
'internal_id': bg._internal_id,
'tags': [],
'provenance': pj}],
'source': ri.provenance.source}
for bg, pj, ri in zip(band_gaps,
provenances_json['band_gaps'],
refractive_indices)
]
self.sq_canonical_sym_json_no_value = copy.deepcopy(self.sq_canonical_sym_as_dicts_no_value)
for sym in ["band_gaps", "refractive_indices"]:
for q, p in zip(self.sq_canonical_sym_json_no_value[sym], provenances_json[sym]):
q['provenance'] = p
self.quantity_with_uncertainty = NumQuantity.from_weighted_mean(b)
self.sq_with_uncertainty_as_dict_no_numbers = {
'@module': 'propnet.dbtools.storage',
'@class': 'StorageQuantity',
'internal_id': self.quantity_with_uncertainty._internal_id,
'data_type': 'NumQuantity',
'symbol_type': symbols['B'],
'units': 'dimensionless',
'provenance': ProvenanceStore.from_provenance_element(
self.quantity_with_uncertainty.provenance),
'tags': []}
provenances_json = {
'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStore',
'model': 'aggregation',
'inputs': [
{'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStoreQuantity',
'data_type': 'NumQuantity',
'symbol_type': self.custom_symbols_json['B'],
'internal_id': b['internal_id'],
'tags': [],
'provenance': b['provenance']}
for b in self.sq_custom_sym_json['B']],
'source': self.quantity_with_uncertainty.provenance.source
}
self.sq_with_uncertainty_json_no_numbers = copy.deepcopy(self.sq_with_uncertainty_as_dict_no_numbers)
self.sq_with_uncertainty_json_no_numbers.update({"symbol_type": self.custom_symbols_json['B'],
"provenance": provenances_json})
self.sq_with_uncertainty_numbers = {"value": 42.0,
"uncertainty": 4.0}
obj_symbol = symbols['C']
self.object_quantity = QuantityFactory.create_quantity(obj_symbol, "Test string")
self.sq_object_as_dict = copy.deepcopy(self.sq_custom_sym_as_dicts['A'][0])
self.sq_object_as_dict.update({
"data_type": "ObjQuantity",
"symbol_type": symbols['C'],
"internal_id": self.object_quantity._internal_id,
"value": "Test string",
"units": None,
"provenance": ProvenanceStore.from_provenance_element(self.object_quantity.provenance)
})
self.sq_object_json = copy.deepcopy(self.sq_object_as_dict)
self.sq_object_json.update(
{"symbol_type": self.custom_syms_as_dicts['C'],
"provenance": {'@module': 'propnet.dbtools.storage',
'@class': 'ProvenanceStore',
'model': None,
'inputs': None,
'source': self.object_quantity.provenance.source}}
)
# This setting allows dict differences to be shown in full
self.maxDiff = None
def process_item(self, item):
# Define quantities corresponding to materials doc fields
# Attach quantities to materials
item = MontyDecoder().process_decoded(item)
logger.info("Populating material for %s", item['task_id'])
material = Material()
if 'created_at' in item.keys():
date_created = item['created_at']
else:
date_created = ""
provenance = ProvenanceElement(
source={
"source": self.source_name,
"source_key": item['task_id'],
"date_created": date_created
})
for mkey, property_name in self.materials_symbol_map.items():
value = get(item, mkey)
if value:
material.add_quantity(
QuantityFactory.create_quantity(property_name,
value,
provenance=provenance))
# Add custom things, e. g. computed entry
computed_entry = get_entry(item)
material.add_quantity(
QuantityFactory.create_quantity("computed_entry",
computed_entry,
provenance=provenance))
material.add_quantity(
QuantityFactory.create_quantity("external_identifier_mp",
item['task_id'],
provenance=provenance))
input_quantities = material.get_quantities()
# Use graph to generate expanded quantity pool
logger.info("Evaluating graph for %s", item['task_id'])
graph = Graph()
graph.remove_models({
"dimensionality_cheon":
DEFAULT_MODEL_DICT['dimensionality_cheon'],
"dimensionality_gorai":
DEFAULT_MODEL_DICT['dimensionality_gorai']
})
new_material = graph.evaluate(material)
# Format document and return
logger.info("Creating doc for %s", item['task_id'])
# Gives the initial inputs that were used to derive properties of a
# certain material.
doc = {
"inputs":
[StorageQuantity.from_quantity(q) for q in input_quantities]
}
for symbol, quantity in new_material.get_aggregated_quantities().items(
):
all_qs = new_material._symbol_to_quantity[symbol]
# Only add new quantities
# TODO: Condition insufficiently general.
# Can end up with initial quantities added as "new quantities"
if len(all_qs) == 1 and list(all_qs)[0] in input_quantities:
continue
# Write out all quantities as dicts including the
# internal ID for provenance tracing
qs = [StorageQuantity.from_quantity(q).as_dict() for q in all_qs]
# THE listing of all Quantities of a given symbol.
sub_doc = {
"quantities": qs,
"mean": unumpy.nominal_values(quantity.value).tolist(),
"std_dev": unumpy.std_devs(quantity.value).tolist(),
"units":
quantity.units.format_babel() if quantity.units else None,
"title": quantity._symbol_type.display_names[0]
}
# Symbol Name -> Sub_Document, listing all Quantities of that type.
doc[symbol.name] = sub_doc
doc.update({
"task_id": item["task_id"],
"pretty_formula": item["pretty_formula"]
})
return jsanitize(doc, strict=True)
def super_evaluate(self, material, allow_model_failure=True):
"""
Given a SuperMaterial object as input, creates a new SuperMaterial
object to include all derivable properties. Returns a reference to
the new, augmented SuperMaterial object.
Args:
material (SuperMaterial): material for which properties
will be expanded.
Returns:
(Material) reference to the newly derived material object.
"""
if not isinstance(material, CompositeMaterial):
raise Exception("material provided is not a SuperMaterial: " +
str(type(material)))
# Evaluate material's sub-materials
evaluated_materials = list()
for m in material.materials:
logger.debug("Evaluating sub-material: " + str(id(m)))
if isinstance(m, CompositeMaterial):
evaluated_materials.append(self.super_evaluate(m))
else:
evaluated_materials.append(self.evaluate(m))
# Run all SuperModels in the graph on this SuperMaterial if
# a material mapping can be established. Store any derived quantities.
all_quantities = defaultdict(set)
for (k, v) in material._symbol_to_quantity:
all_quantities[k].add(v)
to_return = CompositeMaterial(evaluated_materials)
to_return._symbol_to_quantity = all_quantities
logger.debug("Evaluating SuperMaterial")
for model in self._composite_models.values():
logger.debug("\tEvaluating Model: " + model.name)
# Establish material mappings for the given input set.
mat_mappings = model.gen_material_mappings(to_return.materials)
# Avoid ambiguous or impossible mappings, at least for now.
if len(mat_mappings) != 1:
continue
mat_mapping = mat_mappings[0]
# Go through input sets
for property_input_sets in model.evaluation_list:
logger.debug("\t\tGenerating input sets for: " +
str(property_input_sets))
# Create a quantity pool from the appropriate materials.
# Modify inputs for use in generate_input_sets
temp_pool = defaultdict(set)
combined_list = []
mat_list = []
symbol_list = []
for item in property_input_sets:
combined_list.append(item)
mat_list.append(CompositeModel.get_material(item))
symbol_list.append(CompositeModel.get_symbol(item))
for i in range(0, len(mat_list)):
if mat_list[
i] == None: # Draw symbol from the CompositeMaterial
mat = to_return
else:
mat = mat_mapping[mat_list[i]]
for q in mat._symbol_to_quantity[symbol_list[i]]:
temp_pool[combined_list[i]].add(q)
input_sets = self.generate_input_sets(combined_list, temp_pool)
for input_set in input_sets:
logger.debug("\t\t\tEvaluating input set: " +
str(input_set))
# Check if input_set can be evaluated -- input_set must pass the necessary model constraints
if not model.check_constraints(input_set):
logger.debug(
"\t\t\tInput set failed -- did not pass model constraints."
)
continue
# Try to evaluate input_set:
evaluate_set = dict(zip(combined_list, input_set))
output = model.evaluate(evaluate_set,
allow_failure=allow_model_failure)
success = output.pop('successful')
if not success:
logger.debug(
"\t\t\tInput set failed -- did not produce a successful output."
)
continue
# input_set led to output from the Model -- add output to the SuperMaterial
logger.debug("\t\t\tInput set produced successful output.")
for symbol, quantity in output.items():
st = self._symbol_types.get(symbol)
if not st:
raise ValueError(
"Symbol type {} not found".format(symbol))
q = QuantityFactory.create_quantity(st, quantity)
to_return._symbol_to_quantity[st].add(q)
logger.debug("\t\t\tNew output: " + str(q))
# Evaluate the SuperMaterial's quantities and return the result.
mappings = self.evaluate(to_return)._symbol_to_quantity
to_return._symbol_to_quantity = mappings
return to_return
def evaluate(self, symbol_quantity_dict_in, allow_failure=True):
"""
Given a set of property_values, performs error checking to see
if the corresponding input symbol_values represents a valid
input set based on the self.connections() method. If so, returns
a dictionary representing the value of plug_in applied to the
input_symbols. The dictionary contains a "successful" key
representing if plug_in was successful.
The key distinction between evaluate and plug_in is properties
in properties out vs. symbols in symbols out. In addition,
evaluate also handles any requisite unit_mapping
Args:
symbol_quantity_dict ({property_name: Quantity}): a mapping of
symbol names to quantities to be substituted
allow_failure (bool): whether or not to catch
errors in model evaluation
Returns:
dictionary of output properties with associated values
generated from the input, along with "successful" if the
substitution succeeds
"""
# Remap symbols and units if symbol map isn't none
symbol_quantity_dict = self.map_properties_to_symbols(
symbol_quantity_dict_in)
input_symbol_quantity_dict = {k: v for k, v in symbol_quantity_dict.items()
if not (k in self.constraint_symbols
and k not in self.all_input_symbols)}
for (k, v) in symbol_quantity_dict.items():
# replacing = self.symbol_property_map.get(k, k)
replacing = self.symbol_property_map.get(k)
# to_quantity() returns original object if it's already a BaseQuantity
# unlike Quantity() which will return a deep copy
symbol_quantity_dict[k] = QuantityFactory.to_quantity(replacing, v)
# TODO: Is it really necessary to strip these?
# TODO: maybe this only applies to pymodels or things with objects?
# strip units from input and keep for reassignment
symbol_value_dict = {}
for symbol, quantity in symbol_quantity_dict.items():
# If unit map convert and then scrub units
if self.unit_map.get(symbol):
quantity = quantity.to(self.unit_map[symbol])
symbol_value_dict[symbol] = quantity.magnitude
# Otherwise use values
else:
symbol_value_dict[symbol] = quantity.value
contains_complex_input = any(NumQuantity.is_complex_type(v) for v in symbol_value_dict.values())
input_symbol_value_dict = {k: symbol_value_dict[k] for k in input_symbol_quantity_dict.keys()}
# Plug in and check constraints
try:
with PrintToLogger():
out = self.plug_in(input_symbol_value_dict)
except Exception as err:
if allow_failure:
return {"successful": False,
"message": "{} evaluation failed: {}".format(self, err)}
else:
raise err
if not self.check_constraints({**symbol_value_dict, **out}):
return {"successful": False,
"message": "Constraints not satisfied"}
provenance = ProvenanceElement(
model=self.name, inputs=list(input_symbol_quantity_dict.values()),
source="propnet")
out = self.map_symbols_to_properties(out)
unit_map_as_properties = self.map_symbols_to_properties(self.unit_map)
for symbol, value in out.items():
try:
quantity = QuantityFactory.create_quantity(
symbol, value, unit_map_as_properties.get(symbol),
provenance=provenance)
except SymbolConstraintError as err:
if allow_failure:
errmsg = "{} symbol constraint failed: {}".format(self, err)
return {"successful": False,
"message": errmsg}
else:
raise err
if quantity.contains_nan_value():
return {"successful": False,
"message": "Evaluation returned invalid values (NaN)"}
# TODO: Update when we figure out how we're going to handle complex quantities
# Model evaluation will fail if complex values are returned when no complex input was given
# Can surely handle this more gracefully, or assume that the users will apply constraints
if quantity.contains_imaginary_value() and not contains_complex_input:
return {"successful": False,
"message": "Evaluation returned invalid values (complex)"}
out[symbol] = quantity
out['successful'] = True
return out