def get_materials_for_mpids(self, mpids, filter_null_properties=True):
"""
Retrieve a list of Materials from the materials
Project for a given list of Materials Project IDs.
Args:
mpids: a list of Materials Project IDs
Returns:
"""
materials_properties = self.get_properties_for_mpids(
mpids, filter_null_properties=filter_null_properties)
materials = []
for material_properties in materials_properties:
material = Material()
for property_name, property_value in material_properties.items():
provenance = ProvenanceElement(source='Materials Project')
quantity = Quantity(self.mapping[property_name],
property_value,
provenance=provenance)
material.add_quantity(quantity)
materials.append(material)
return materials
def evaluate(input_rows):
quantities = [
Quantity(symbol_type=ROW_IDX_TO_SYMBOL_NAME[idx],
value=ureg.parse_expression(row['Value']))
for idx, row in enumerate(input_rows) if row['Value']
]
material = Material()
for quantity in quantities:
material.add_quantity(quantity)
graph = Graph()
output_material = graph.evaluate(material)
output_quantities = output_material.get_aggregated_quantities()
print(output_quantities)
output_rows = [{
'Property': symbol.display_names[0],
'Value': str(quantity.value),
'Provenance': None
} for symbol, quantity in output_quantities.items()]
return output_rows
def evaluate(input_rows, data, aggregate):
quantities = [
QuantityFactory.create_quantity(
symbol_type=ROW_IDX_TO_SYMBOL_NAME[idx],
value=ureg.parse_expression(row['Editable Value']))
for idx, row in enumerate(input_rows) if row['Editable Value']
]
if data and len(data) > 0:
quantities += json.loads(data, cls=MontyDecoder).values()
if not quantities:
raise PreventUpdate
material = Material()
for quantity in quantities:
material.add_quantity(quantity)
graph = Graph()
output_material = graph.evaluate(material)
if aggregate:
output_quantities = output_material.get_aggregated_quantities(
).values()
else:
output_quantities = output_material.get_quantities()
output_rows = [{
'Property': quantity.symbol.display_names[0],
'Value': quantity.pretty_string(sigfigs=3)
} for quantity in output_quantities]
output_table = dt.DataTable(id='output-table',
rows=output_rows,
editable=False)
# TODO: clean up
input_quantity_names = [q.symbol.name for q in quantities]
derived_quantity_names = set(
[q.symbol.name for q in output_quantities]) - \
set(input_quantity_names)
material_graph_data = graph_conversion(
graph.get_networkx_graph(),
nodes_to_highlight_green=input_quantity_names,
nodes_to_highlight_yellow=list(derived_quantity_names))
options = AESTHETICS['global_options']
options['edges']['color'] = '#000000'
output_graph = html.Div(GraphComponent(id='material-graph',
graph=material_graph_data,
options=options),
style={
'width': '100%',
'height': '400px'
})
return [output_graph, html.Br(), output_table]
def generate_canonical_material(c_symbols):
"""
Generates a Material with appropriately attached Quantities.
Args:
c_symbols: (dict<str, Symbol>) dictionary of defined materials.
Returns:
(Material) material with properties loaded.
"""
q1 = Quantity(c_symbols['A'], 19)
q2 = Quantity(c_symbols['A'], 23)
m = Material()
m.add_quantity(q1)
m.add_quantity(q2)
return m
def test_evaluate_single_material_degenerate_property(self):
"""
Graph has one material on it: mat1
mat1 has trivial degenerate properties relative permittivity and relative permeability
2 experimental relative permittivity measurements
2 experimental relative permeability measurements
Determines if TestGraph1 is correctly evaluated using the evaluate method.
We expect 4 refractive_index properties to be calculated as the following:
sqrt(3), sqrt(5), sqrt(6), sqrt(10)
"""
# Setup
propnet = Graph()
mat1 = Material()
mat1.add_quantity(Quantity(DEFAULT_SYMBOLS['relative_permeability'], 1))
mat1.add_quantity(Quantity(DEFAULT_SYMBOLS['relative_permeability'], 2))
mat1.add_quantity(Quantity(DEFAULT_SYMBOLS['relative_permittivity'], 3))
mat1.add_quantity(Quantity(DEFAULT_SYMBOLS['relative_permittivity'], 5))
mat1_derived = propnet.evaluate(mat1)
# Expected outputs
s_outputs = []
s_outputs.append(Quantity('relative_permeability', 1))
s_outputs.append(Quantity('relative_permeability', 2))
s_outputs.append(Quantity('relative_permittivity', 3))
s_outputs.append(Quantity('relative_permittivity', 5))
s_outputs.append(Quantity('refractive_index', 3 ** 0.5))
s_outputs.append(Quantity('refractive_index', 5 ** 0.5))
s_outputs.append(Quantity('refractive_index', 6 ** 0.5))
s_outputs.append(Quantity('refractive_index', 10 ** 0.5))
st_outputs = []
st_outputs.append(DEFAULT_SYMBOLS['relative_permeability'])
st_outputs.append(DEFAULT_SYMBOLS['relative_permittivity'])
st_outputs.append(DEFAULT_SYMBOLS['refractive_index'])
# Test
for q_expected in s_outputs:
q = None
for q_derived in mat1_derived._symbol_to_quantity[q_expected.symbol]:
if q_derived == q_expected:
q = q_derived
self.assertTrue(q is not None,
"Quantity missing from evaluate.")
def get_materials_for_mpids(self, mpids, filter_null_values=True):
"""
Retrieve a list of Materials from the materials
Project for a given list of Materials Project IDs.
Args:
mpids: a list of Materials Project IDs
Returns:
"""
materials_quantities = self.get_quantities_for_mpids(
mpids,
filter_null_values=filter_null_values,
include_date_created=True)
materials = []
for material_quantities in materials_quantities:
material = Material()
try:
date_created = material_quantities.pop('created_at')
except KeyError:
date_created = None
for symbol_name, value in material_quantities.items():
provenance = ProvenanceElement(
source={
'source': 'Materials Project',
'source_key': material_quantities.get(
'material_id', None),
'date_created': date_created
})
quantity = QuantityFactory.create_quantity(
self.mapping[symbol_name],
value,
units=Registry("units").get(self.mapping[symbol_name],
None),
provenance=provenance)
material.add_quantity(quantity)
materials.append(material)
return materials
def process_item(self, item):
# Define quantities corresponding to materials doc fields
# Attach quantities to materials
item = MontyDecoder().process_decoded(item)
logger.info("Populating material for %s", item['task_id'])
material = Material()
for mkey, property_name in self.materials_symbol_map.items():
value = get(item, mkey)
if value:
material.add_quantity(Quantity(property_name, value))
# Add custom things, e. g. computed entry
computed_entry = get_entry(item)
material.add_quantity(Quantity("computed_entry", computed_entry))
material.add_quantity(
Quantity("external_identifier_mp", item['task_id']))
input_quantities = material.get_quantities()
# Use graph to generate expanded quantity pool
logger.info("Evaluating graph for %s", item['task_id'])
graph = Graph()
graph.remove_models({
"dimensionality_cheon":
DEFAULT_MODEL_DICT['dimensionality_cheon'],
"dimensionality_gorai":
DEFAULT_MODEL_DICT['dimensionality_gorai']
})
new_material = graph.evaluate(material)
# Format document and return
logger.info("Creating doc for %s", item['task_id'])
doc = {"inputs": [quantity.as_dict() for quantity in input_quantities]}
for symbol, quantity in new_material.get_aggregated_quantities().items(
):
all_qs = new_material._symbol_to_quantity[symbol]
# Only add new quantities
if len(all_qs) == 1 and list(all_qs)[0] in input_quantities:
continue
qs = [quantity.as_dict() for quantity in all_qs]
sub_doc = {
"quantities": qs,
"mean": unumpy.nominal_values(quantity.value).tolist(),
"std_dev": unumpy.std_devs(quantity.value).tolist(),
"units": qs[0]['units'],
"title": quantity._symbol_type.display_names[0]
}
doc[symbol.name] = sub_doc
doc.update({
"task_id": item["task_id"],
"pretty_formula": item["pretty_formula"]
})
return jsanitize(doc, strict=True)
def process(self, item):
if self.graph_parallel and not self.allow_child_process and \
current_process().name != "MainProcess":
logger.warning(
"It appears derive_quantities() is running "
"in a child process, possibly in a parallelized "
"Runner.\nThis is not recommended and will deteriorate "
"performance.")
# Define quantities corresponding to materials doc fields
# Attach quantities to materials
item = MontyDecoder().process_decoded(item)
logger.info("Populating material for %s", item['task_id'])
material = Material()
if 'created_at' in item.keys():
date_created = item['created_at']
else:
date_created = None
provenance = ProvenanceElement(
source={
"source": self.source_name,
"source_key": item['task_id'],
"date_created": date_created
})
for mkey, property_name in self.materials_symbol_map.items():
value = pydash.get(item, mkey)
if value:
material.add_quantity(
QuantityFactory.create_quantity(
property_name,
value,
units=Registry("units").get(property_name, None),
provenance=provenance))
# Add custom things, e. g. computed entry
computed_entry = get_entry(item)
if computed_entry:
material.add_quantity(
QuantityFactory.create_quantity("computed_entry",
computed_entry,
provenance=provenance))
else:
logger.info("Unable to create computed entry for {}".format(
item['task_id']))
material.add_quantity(
QuantityFactory.create_quantity("external_identifier_mp",
item['task_id'],
provenance=provenance))
input_quantities = material.symbol_quantities_dict
# Use graph to generate expanded quantity pool
logger.info("Evaluating graph for %s", item['task_id'])
new_material = self._graph_evaluator.evaluate(
material, timeout=self.graph_timeout)
# Format document and return
logger.info("Creating doc for %s", item['task_id'])
# Gives the initial inputs that were used to derive properties of a
# certain material.
doc = {
"inputs": [
StorageQuantity.from_quantity(q)
for q in chain.from_iterable(input_quantities.values())
]
}
for symbol, quantities in new_material.symbol_quantities_dict.items():
# If no new quantities of a given symbol were derived (i.e. if the initial
# input quantity/ies is/are the only one/s listed in the new material) then don't add
# that quantity to the propnet entry document as a derived quantity.
if len(quantities) == len(input_quantities[symbol]):
continue
sub_doc = {}
try:
# Write out all quantities as dicts including the
# internal ID for provenance tracing
qs = [
jsanitize(StorageQuantity.from_quantity(q), strict=True)
for q in quantities
]
except AttributeError as ex:
# Check to see if this is an error caused by an object
# that is not JSON serializable
msg = ex.args[0]
if "object has no attribute 'as_dict'" in msg:
# Write error to db and logger
errmsg = "Quantity of Symbol '{}' is not ".format(symbol.name) + \
"JSON serializable. Cannot write quantities to database!"
logger.error(errmsg)
sub_doc['error'] = errmsg
qs = []
else:
# If not, re-raise the error
raise ex
sub_doc['quantities'] = qs
doc[symbol.name] = sub_doc
aggregated_quantities = new_material.get_aggregated_quantities()
for symbol, quantity in aggregated_quantities.items():
if symbol.name not in doc:
# No new quantities were derived
continue
# Store mean and std dev for aggregated quantities
sub_doc = {
"mean": unumpy.nominal_values(quantity.value).tolist(),
"std_dev": unumpy.std_devs(quantity.value).tolist(),
"units":
quantity.units.format_babel() if quantity.units else None,
"title": quantity.symbol.display_names[0]
}
# Symbol Name -> Sub_Document, listing all Quantities of that type.
doc[symbol.name].update(sub_doc)
doc.update({
"task_id": item["task_id"],
"pretty_formula": item.get("pretty_formula"),
"deprecated": item.get("deprecated", False)
})
if self.include_sandboxed:
doc.update({'sbxn': item.get("sbxn", [])})
return jsanitize(doc, strict=True)
def process_item(self, item):
# Define quantities corresponding to materials doc fields
# Attach quantities to materials
item = MontyDecoder().process_decoded(item)
logger.info("Populating material for %s", item['task_id'])
material = Material()
if 'created_at' in item.keys():
date_created = item['created_at']
else:
date_created = ""
provenance = ProvenanceElement(
source={
"source": self.source_name,
"source_key": item['task_id'],
"date_created": date_created
})
for mkey, property_name in self.materials_symbol_map.items():
value = get(item, mkey)
if value:
material.add_quantity(
QuantityFactory.create_quantity(property_name,
value,
provenance=provenance))
# Add custom things, e. g. computed entry
computed_entry = get_entry(item)
material.add_quantity(
QuantityFactory.create_quantity("computed_entry",
computed_entry,
provenance=provenance))
material.add_quantity(
QuantityFactory.create_quantity("external_identifier_mp",
item['task_id'],
provenance=provenance))
input_quantities = material.get_quantities()
# Use graph to generate expanded quantity pool
logger.info("Evaluating graph for %s", item['task_id'])
graph = Graph()
graph.remove_models({
"dimensionality_cheon":
DEFAULT_MODEL_DICT['dimensionality_cheon'],
"dimensionality_gorai":
DEFAULT_MODEL_DICT['dimensionality_gorai']
})
new_material = graph.evaluate(material)
# Format document and return
logger.info("Creating doc for %s", item['task_id'])
# Gives the initial inputs that were used to derive properties of a
# certain material.
doc = {
"inputs":
[StorageQuantity.from_quantity(q) for q in input_quantities]
}
for symbol, quantity in new_material.get_aggregated_quantities().items(
):
all_qs = new_material._symbol_to_quantity[symbol]
# Only add new quantities
# TODO: Condition insufficiently general.
# Can end up with initial quantities added as "new quantities"
if len(all_qs) == 1 and list(all_qs)[0] in input_quantities:
continue
# Write out all quantities as dicts including the
# internal ID for provenance tracing
qs = [StorageQuantity.from_quantity(q).as_dict() for q in all_qs]
# THE listing of all Quantities of a given symbol.
sub_doc = {
"quantities": qs,
"mean": unumpy.nominal_values(quantity.value).tolist(),
"std_dev": unumpy.std_devs(quantity.value).tolist(),
"units":
quantity.units.format_babel() if quantity.units else None,
"title": quantity._symbol_type.display_names[0]
}
# Symbol Name -> Sub_Document, listing all Quantities of that type.
doc[symbol.name] = sub_doc
doc.update({
"task_id": item["task_id"],
"pretty_formula": item["pretty_formula"]
})
return jsanitize(doc, strict=True)
class MaterialTest(unittest.TestCase):
def setUp(self):
# Create some test properties and a few base objects
self.q1 = Quantity(
DEFAULT_SYMBOLS['bulk_modulus'],
ureg.Quantity.from_tuple([200, [['gigapascals', 1]]]))
self.q2 = Quantity(
DEFAULT_SYMBOLS['shear_modulus'],
ureg.Quantity.from_tuple([100, [['gigapascals', 1]]]))
self.q3 = Quantity(
DEFAULT_SYMBOLS['bulk_modulus'],
ureg.Quantity.from_tuple([300, [['gigapascals', 1]]]))
self.material = Material()
self.graph = Graph()
def test_material_setup(self):
self.assertTrue(
len(self.material._symbol_to_quantity) == 0,
"Material not initialized properly.")
def test_material_add_quantity(self):
self.material.add_quantity(self.q1)
self.assertEqual(len(self.material._symbol_to_quantity), 1,
"Material did not add the quantity.")
self.material.add_quantity(self.q2)
self.assertEqual(len(self.material._symbol_to_quantity), 2,
"Material did not add quantity to itself.")
def test_material_remove_quantity(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.remove_quantity(self.q1)
self.assertEqual(
len(self.material._symbol_to_quantity[
DEFAULT_SYMBOLS['shear_modulus']]), 1,
"Material did not remove the correct quantity.")
self.material.remove_quantity(self.q2)
self.assertEqual(
len(self.material._symbol_to_quantity[
DEFAULT_SYMBOLS['shear_modulus']]), 0,
"Material did not remove the quantity correctly.")
def test_material_remove_symbol(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
self.material.remove_symbol(DEFAULT_SYMBOLS['bulk_modulus'])
self.assertTrue(
DEFAULT_SYMBOLS['shear_modulus']
in self.material._symbol_to_quantity.keys(),
"Material did not remove Symbol correctly.")
self.assertTrue(
self.q2 in self.material._symbol_to_quantity[
DEFAULT_SYMBOLS['shear_modulus']],
"Material did not remove Symbol correctly.")
self.assertEqual(len(self.material._symbol_to_quantity), 1,
"Material did not remove Symbol correctly.")
def test_get_symbols(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
out = self.material.get_symbols()
self.assertEqual(len(out), 2,
"Material did not get Symbol Types correctly.")
self.assertTrue(DEFAULT_SYMBOLS['bulk_modulus'] in out,
"Material did not get Symbol Types correctly.")
self.assertTrue(DEFAULT_SYMBOLS['shear_modulus'] in out,
"Material did not get Symbol Types correctly.")
def test_get_quantities(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
out = self.material.get_quantities()
self.assertTrue(all([q in out for q in [self.q1, self.q2, self.q3]]),
"Material did not get Quantity objects correctly.")
def test_get_aggregated_quantities(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
agg = self.material.get_aggregated_quantities()
class MaterialTest(unittest.TestCase):
def setUp(self):
# Create some test properties and a few base objects
self.q1 = QuantityFactory.create_quantity(DEFAULT_SYMBOLS['bulk_modulus'],
ureg.Quantity.from_tuple([200, [['gigapascals', 1]]]))
self.q2 = QuantityFactory.create_quantity(DEFAULT_SYMBOLS['shear_modulus'],
ureg.Quantity.from_tuple([100, [['gigapascals', 1]]]))
self.q3 = QuantityFactory.create_quantity(DEFAULT_SYMBOLS['bulk_modulus'],
ureg.Quantity.from_tuple([300, [['gigapascals', 1]]]))
self.material = Material()
self.graph = Graph()
def test_material_setup(self):
self.assertTrue(len(self.material._symbol_to_quantity) == 0,
"Material not initialized properly.")
def test_material_add_quantity(self):
self.material.add_quantity(self.q1)
self.assertEqual(len(self.material._symbol_to_quantity), 1,
"Material did not add the quantity.")
self.material.add_quantity(self.q2)
self.assertEqual(len(self.material._symbol_to_quantity), 2,
"Material did not add quantity to itself.")
def test_material_remove_quantity(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.remove_quantity(self.q1)
self.assertEqual(
len(self.material._symbol_to_quantity[DEFAULT_SYMBOLS['shear_modulus']]),
1, "Material did not remove the correct quantity.")
self.material.remove_quantity(self.q2)
self.assertEqual(
len(self.material._symbol_to_quantity[DEFAULT_SYMBOLS['shear_modulus']]),
0, "Material did not remove the quantity correctly.")
def test_material_remove_symbol(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
self.material.remove_symbol(DEFAULT_SYMBOLS['bulk_modulus'])
self.assertTrue(
DEFAULT_SYMBOLS['shear_modulus'] in self.material._symbol_to_quantity.keys(),
"Material did not remove Symbol correctly.")
self.assertTrue(
self.q2 in self.material._symbol_to_quantity[DEFAULT_SYMBOLS['shear_modulus']],
"Material did not remove Symbol correctly.")
self.assertEqual(len(self.material._symbol_to_quantity), 1,
"Material did not remove Symbol correctly.")
def test_get_symbols(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
out = self.material.get_symbols()
self.assertEqual(
len(out), 2, "Material did not get Symbol Types correctly.")
self.assertTrue(DEFAULT_SYMBOLS['bulk_modulus'] in out,
"Material did not get Symbol Types correctly.")
self.assertTrue(DEFAULT_SYMBOLS['shear_modulus'] in out,
"Material did not get Symbol Types correctly.")
def test_get_quantities(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
out = self.material.get_quantities()
self.assertTrue(all([q in out for q in [self.q1, self.q2, self.q3]]),
"Material did not get Quantity objects correctly.")
def test_get_aggregated_quantities(self):
self.material.add_quantity(self.q1)
self.material.add_quantity(self.q2)
self.material.add_quantity(self.q3)
agg = self.material.get_aggregated_quantities()
# TODO: add a meaningful test here
def test_add_default_quantities(self):
material = Material(add_default_quantities=True)
self.assertEqual(list(material['temperature'])[0],
QuantityFactory.create_quantity("temperature", 300,
provenance=ProvenanceElement(model='default')))
self.assertEqual(list(material['relative_permeability'])[0],
QuantityFactory.create_quantity("relative_permeability", 1,
provenance=ProvenanceElement(model='default')))
def evaluate(input_rows, data, aggregate):
quantities = []
for idx, row in enumerate(input_rows):
if row['Editable Value']:
try:
value = ureg.parse_expression(row['Editable Value'])
units = Registry("units").get(ROW_IDX_TO_SYMBOL_NAME[idx])
value.ito(units)
except Exception:
# Someone put an invalid value in the table
# TODO: Make error known to the user
raise PreventUpdate
q = QuantityFactory.create_quantity(
symbol_type=ROW_IDX_TO_SYMBOL_NAME[idx], value=value)
quantities.append(q)
if data and len(data) > 0:
quantities += json.loads(data, cls=MontyDecoder).values()
if not quantities:
raise PreventUpdate
material = Material()
for quantity in quantities:
material.add_quantity(quantity)
output_material = graph_evaluator.evaluate(material, timeout=5)
if aggregate:
aggregated_quantities = output_material.get_aggregated_quantities()
non_aggregatable_quantities = [
v for v in output_material.get_quantities()
if v.symbol not in aggregated_quantities
]
output_quantities = list(
aggregated_quantities.values()) + non_aggregatable_quantities
else:
output_quantities = output_material.get_quantities()
output_rows = [{
'Property': quantity.symbol.display_names[0],
'Value': quantity.pretty_string(sigfigs=3)
} for quantity in output_quantities]
output_table = dt.DataTable(id='output-table',
data=output_rows,
columns=[{
'id': val,
'name': val
} for val in ('Property', 'Value')],
editable=False,
**DATA_TABLE_STYLE)
# TODO: clean up
input_quantity_names = [q.symbol for q in quantities]
derived_quantity_names = \
set([q.symbol for q in output_quantities]) - \
set(input_quantity_names)
models_evaluated = set(
output_q.provenance.model
for output_q in output_material.get_quantities())
models_evaluated = [
Registry("models").get(m) for m in models_evaluated
if Registry("models").get(m) is not None
]
material_graph_data = graph_conversion(
propnet_nx_graph,
derivation_pathway={
'inputs': input_quantity_names,
'outputs': list(derived_quantity_names),
'models': models_evaluated
})
output_graph = html.Div(children=[
dcc.Checklist(id='material-graph-options',
options=[{
'label': 'Show models',
'value': 'show_models'
}, {
'label': 'Show properties',
'value': 'show_properties'
}],
value=['show_properties'],
labelStyle={'display': 'inline-block'}),
Cytoscape(id='material-graph',
elements=material_graph_data,
stylesheet=GRAPH_STYLESHEET,
layout=GRAPH_LAYOUT_CONFIG,
**GRAPH_SETTINGS['full_view'])
])
return [output_graph, html.Br(), output_table]