def add_es(mat, new_style_mat): bs_origin = None dos_origin = None try: bs_origin = next( ( origin for origin in new_style_mat.get("origins", []) if "Line" in origin["task_type"] ), None, ) dos_origin = next( ( origin for origin in new_style_mat.get("origins", []) if "Uniform" in origin["task_type"] ), None, ) if bs_origin: u_type = "GGA+U" if "+U" in bs_origin["task_type"] else "GGA" set_(mat, "band_structure.{}.task_id".format(u_type), bs_origin["task_id"]) if dos_origin: u_type = "GGA+U" if "+U" in dos_origin["task_type"] else "GGA" set_(mat, "dos.{}.task_id".format(u_type), dos_origin["task_id"]) except Exception as e: print("Error in adding electronic structure: {}".format(e)) mat["has_bandstructure"] = "bandstructure" in new_style_mat.get("has",[])
def test_submit_form_with_update_no_login_required(self): """Submit form.""" responseId, submit_res = self.submit_form(self.formId, ONE_FORMDATA) self.assertEqual(submit_res.pop("value"), ONE_FORMDATA) self.assertEqual(submit_res, ONE_SUBMITRES, submit_res) self.assertIn("paymentMethods", submit_res) """View response.""" response = self.view_response(responseId) self.assertEqual(response["value"], ONE_FORMDATA) self.assertEqual(response["paid"], False) self.assertTrue(response.get("user", None) == None) expected_data = copy.deepcopy(ONE_FORMDATA) set_(expected_data, "contact_name.last", "NEW_LAST2") response = self.lg.handle_request( method="POST", path=f"/forms/{self.formId}", headers={ "authorization": "auth", "Content-Type": "application/json" }, body=json.dumps({ "data": expected_data, "responseId": responseId }), ) self.assertEqual(response["statusCode"], 200, response) response = self.view_response(responseId) self.assertEqual(response["value"], expected_data) self.assertEqual(response["paid"], False)
def make_mat(self, task_group): """ Converts a group of tasks into one material """ all_props = list(chain.from_iterable( [self.task_to_prop_list(t) for t in task_group])) sorted_props = sorted(all_props, key=lambda x: x['materials_key']) grouped_props = groupby(sorted_props, lambda x: x['materials_key']) best_props = [] for name, prop in grouped_props: sorted_props = sorted(prop, key=lambda x: x[ 'quality_score'], reverse=True) best_props.append(sorted_props[0]) # Add in the provenance for the properties origins = [{k: prop[k] for k in ["materials_key", "task_type", "task_id", "last_updated"]} for prop in best_props if prop.get("track", False)] task_ids = list(sorted([t["task_id"] for t in task_group], key=lambda x: int(str(x).split("-")[-1]))) task_types = {t["task_id"] : t["task_type"] for t in all_props} mat = {"updated_at": datetime.utcnow(), "task_ids": task_ids, self.materials.key: task_ids[0], "origins": origins, "task_types": task_types } for prop in best_props: set_(mat, prop["materials_key"], prop["value"]) return mat
def substitute(d: Dict, aliases: Dict): """ Substitutes keys in dictionary Accepts multilevel mongo like keys """ for alias, key in aliases.items(): if has(d, key): set_(d, alias, get(d, key)) unset(d, key)
def old_style_mat(new_style_mat): """ Creates the base document for the old MP mapidoc style from the new document structure """ mat = {} for mp, new_key in mp_conversion_dict.items(): if has(new_style_mat, new_key): set_(mat, mp, get(new_style_mat, new_key)) mat["is_ordered"] = True mat["is_compatible"] = True struc = Structure.from_dict(mat["structure"]) mat["oxide_type"] = oxide_type(struc) mat["reduced_cell_formula"] = struc.composition.reduced_composition.as_dict() mat["unit_cell_formula"] = struc.composition.as_dict() mat["full_formula"] = "".join(struc.formula.split()) vals = sorted(mat["reduced_cell_formula"].values()) mat["anonymous_formula"] = {string.ascii_uppercase[i]: float(vals[i]) for i in range(len(vals))} mat["initial_structure"] = new_style_mat.get("initial_structure", None) mat["nsites"] = struc.get_primitive_structure().num_sites set_(mat, "pseudo_potential.functional", "PBE") set_(mat, "pseudo_potential.labels", [p["titel"].split()[1] for p in get(new_style_mat, "calc_settings.potcar_spec")]) mat["ntask_ids"] = len(get(new_style_mat, "task_ids")) set_(mat, "pseudo_potential.pot_type", "paw") add_blessed_tasks(mat, new_style_mat) add_cifs(mat) check_relaxation(mat, new_style_mat) return mat
def update_response_path(response, path, value, response_base_path): from ..main import app existing_value = get(getattr(response, response_base_path), path, value) if type(value) is str: if type(existing_value) is bool and value.lower() in ("true", "false"): value = value.lower() == "true" elif type(existing_value) is float and value.isdigit(): value = float(value) elif type(existing_value) is int and value.isdigit(): value = int(value) set_(getattr(response, response_base_path), path, value) response.update_trail.append( UpdateTrailItem(path=path, old_value=existing_value, new_value=value, date=datetime.datetime.now(), user=app.get_current_user_id(), response_base_path=response_base_path))
def old_style_mat(new_style_mat): """ Creates the base document for the old MP mapidoc style from the new document structure """ mat = {} mp_conversion_dict = _settings["conversion_dict"] mag_types = _settings["mag_types"] # Uses the conversion dict to copy over values which handles the bulk of the work. for mp, new_key in mp_conversion_dict.items(): if has(new_style_mat, new_key): set_(mat, mp, get(new_style_mat, new_key)) # Anything coming through DFT is always ordered mat["is_ordered"] = True mat["is_compatible"] = True struc = Structure.from_dict(mat["structure"]) mat["oxide_type"] = oxide_type(struc) mat["reduced_cell_formula"] = struc.composition.reduced_composition.as_dict( ) mat["unit_cell_formula"] = struc.composition.as_dict() mat["full_formula"] = "".join(struc.formula.split()) vals = sorted(mat["reduced_cell_formula"].values()) mat["anonymous_formula"] = { string.ascii_uppercase[i]: float(vals[i]) for i in range(len(vals)) } mat["initial_structure"] = new_style_mat.get("initial_structure", None) mat["nsites"] = struc.get_primitive_structure().num_sites set_(mat, "pseudo_potential.functional", "PBE") set_( mat, "pseudo_potential.labels", [ p["titel"].split()[1] for p in get(new_style_mat, "calc_settings.potcar_spec") ], ) set_(mat, "pseudo_potential.pot_type", "paw") mat["blessed_tasks"] = { d["task_type"]: d["task_id"] for d in new_style_mat["origins"] } mat["deprecated_tasks"] = new_style_mat.get("deprecated_tasks", []) mat["ntask_ids"] = len(mat["task_ids"]) return mat
def add_param_overrides(self, data, **kwargs): # Removing params override from context so that overriding is done once on the yaml # child schema should not override the params. params_override = self.context.pop(PARAMS_OVERRIDE_KEY, None) if params_override is not None: for override in params_override: for param, val in override.items(): # Check that none of the intermediary levels are string references (azureml/file) param_tokens = param.split(".") test_layer = data for layer in param_tokens: if test_layer is None: continue if isinstance(test_layer, str): raise Exception( f"Cannot use '--set' on properties defined by reference strings: --set {param}" ) test_layer = test_layer.get(layer, None) objects.set_(data, param, val) return data
def add_es(mat, new_mat, es): bs_origin = None dos_origin = None try: bs_origin = next((origin for origin in new_mat.get("origins", []) if "Line" in origin["task_type"]), None) dos_origin = next((origin for origin in new_mat.get("origins", []) if "Uniform" in origin["task_type"]), None) if bs_origin: u_type = "GGA+U" if "+U" in bs_origin["task_type"] else "GGA" set_(mat, "band_structure.{}.task_id".format(u_type), bs_origin["task_id"]) if has(es, "band_gap"): set_(mat, "band_gap.search_gap", get(es, "band_gap")) if dos_origin: u_type = "GGA+U" if "+U" in dos_origin["task_type"] else "GGA" set_(mat, "dos.{}.task_id".format(u_type), dos_origin["task_id"]) except Exception as e: print("Error in adding electronic structure: {}".format(e)) mat["has_bandstructure"] = bool(bs_origin) and bool(dos_origin)
def add_thermo(mat, thermo): if has(thermo, "thermo.e_above_hull"): set_(mat, "e_above_hull", get(thermo, "thermo.e_above_hull")) if has(thermo, "thermo.formation_energy_per_atom"): set_(mat, "formation_energy_per_atom", get(thermo, "thermo.formation_energy_per_atom")) if has(thermo, "thermo.decomposes_to"): set_(mat, "decomposes_to", get(thermo, "thermo.decomposes_to"))
def convert_mpworks_to_atomate(mpworks_doc, update_mpworks=True): """ Function to convert an mpworks document into an atomate document, uses schema above and a few custom cases Args: mpworks_doc (dict): mpworks task document update_mpworks (bool): flag to indicate that mpworks schema should be updated to final MPWorks version """ if update_mpworks: update_mpworks_schema(mpworks_doc) atomate_doc = {} for key_mpworks, key_atomate in settings['task_conversion_keys'].items(): val = get(mpworks_doc, key_mpworks) set_(atomate_doc, key_atomate, val) # Task type atomate_doc["task_label"] = settings['task_label_conversions'].get( mpworks_doc["task_type"]) # calculations atomate_doc["calcs_reversed"] = mpworks_doc["calculations"][::-1] # anonymous formula comp = Composition(atomate_doc['composition_reduced']) atomate_doc["formula_anonymous"] = comp.anonymized_formula # deformation matrix and original_task_id if "deformation_matrix" in mpworks_doc: # Transpose this b/c of old bug, should verify in doc processing defo = mpworks_doc["deformation_matrix"] if isinstance(defo, str): defo = convert_string_deformation_to_list(defo) defo = np.transpose(defo).tolist() set_(atomate_doc, "transmuter.transformations", ["DeformStructureTransformation"]) set_(atomate_doc, "transmuter.transformation_params", [{ "deformation": defo }]) return atomate_doc
def test_edit_response(self): """Create form.""" self.formId = self.create_form() self.edit_form( self.formId, { "schema": ONE_SCHEMA, "uiSchema": ONE_UISCHEMA, "formOptions": dict(ONE_FORMOPTIONS, loginRequired=True) }) """Submit form.""" responseId, submit_res = self.submit_form(self.formId, ONE_FORMDATA) self.assertEqual(submit_res.pop("value"), ONE_FORMDATA) self.assertEqual(submit_res, ONE_SUBMITRES, submit_res) self.assertIn("paymentMethods", submit_res) """View response.""" response = self.view_response(responseId) self.assertEqual(response['value'], ONE_FORMDATA) responseIdNew, submit_res = self.submit_form(self.formId, ONE_FORMDATA, responseId) self.assertEqual(responseIdNew, responseId) self.assertEqual(submit_res.pop("value"), ONE_FORMDATA) self.assertEqual( submit_res, { 'paid': False, 'amt_received': { 'currency': 'USD', 'total': 0.0 }, 'success': True, 'action': 'update', 'email_sent': False, 'paymentInfo': { 'currency': 'USD', 'items': [{ 'amount': 0.5, 'description': 'Base Registration', 'name': 'Base Registration', 'quantity': 1.0 }], 'total': 0.5 }, 'paymentMethods': { 'paypal_classic': { 'address1': '123', 'address2': 'asdad', 'business': '*****@*****.**', 'city': 'Atlanta', 'cmd': '_cart', 'email': '*****@*****.**', 'first_name': 'Ashwin', 'image_url': 'http://www.chinmayanewyork.org/wp-content/uploads/2014/08/banner17_ca1.png', 'last_name': 'Ash', 'payButtonText': 'Pay Now', 'sandbox': False, 'state': 'GA', 'zip': '30022' } } }) """Edit response.""" body = {"path": "contact_name.last", "value": "NEW_LAST!"} response = self.lg.handle_request(method='PATCH', path=f'/responses/{responseId}', headers={ "authorization": "auth", "Content-Type": "application/json" }, body=json.dumps(body)) expected_data = copy.deepcopy(ONE_FORMDATA) set_(expected_data, "contact_name.last", "NEW_LAST!") self.assertEqual(response['statusCode'], 200, response) body = json.loads(response['body']) self.assertEqual(body['res']['response']['value'], expected_data) self.assertEqual( [remove_date(i) for i in body['res']['response']['update_trail']], [{ 'old': { 'contact_name': { 'first': 'Ashwin', 'last': 'Ash' }, 'address': { 'line1': '123', 'line2': 'asdad', 'city': 'Atlanta', 'state': 'GA', 'zipcode': '30022' }, 'email': '*****@*****.**', 'phone': '1231231233', 'amount': 0.5, 'subscribe': True }, 'new': { 'contact_name': { 'first': 'Ashwin', 'last': 'Ash' }, 'address': { 'line1': '123', 'line2': 'asdad', 'city': 'Atlanta', 'state': 'GA', 'zipcode': '30022' }, 'email': '*****@*****.**', 'phone': '1231231233', 'amount': 0.5, 'subscribe': True }, 'update_type': 'update', '_cls': 'chalicelib.models.UpdateTrailItem' }, { 'path': 'contact_name.last', 'user': '******', 'old_value': 'Ash', 'new_value': 'NEW_LAST!', 'response_base_path': 'value', '_cls': 'chalicelib.models.UpdateTrailItem' }])
def test_edit_response(self): """Create form.""" self.formId = self.create_form() self.edit_form( self.formId, { "schema": ONE_SCHEMA, "uiSchema": ONE_UISCHEMA, "formOptions": dict(ONE_FORMOPTIONS, loginRequired=True), }, ) """Submit form.""" responseId, submit_res = self.submit_form(self.formId, ONE_FORMDATA) self.assertEqual(submit_res.pop("value"), ONE_FORMDATA) self.assertEqual(submit_res, ONE_SUBMITRES, submit_res) self.assertIn("paymentMethods", submit_res) """View response.""" response = self.view_response(responseId) self.assertEqual(response["value"], ONE_FORMDATA) responseIdNew, submit_res = self.submit_form(self.formId, ONE_FORMDATA, responseId) self.assertEqual(responseIdNew, responseId) self.assertEqual(submit_res.pop("value"), ONE_FORMDATA) self.assertEqual( submit_res, { "paid": False, "amt_received": { "currency": "USD", "total": 0.0 }, "success": True, "action": "update", "email_sent": False, "paymentInfo": { "currency": "USD", "items": [{ "amount": 0.5, "description": "Base Registration", "name": "Base Registration", "quantity": 1.0, "total": 0.5, }], "total": 0.5, }, "amount_owed_cents": 50, "paymentMethods": { "paypal_classic": { "address1": "123", "address2": "asdad", "business": "*****@*****.**", "city": "Atlanta", "cmd": "_cart", "email": "*****@*****.**", "first_name": "Ashwin", "image_url": "http://www.chinmayanewyork.org/wp-content/uploads/2014/08/banner17_ca1.png", "last_name": "Ash", "payButtonText": "Pay Now", "sandbox": False, "state": "GA", "zip": "30022", } }, }, ) """Edit response.""" body = {"path": "contact_name.last", "value": "NEW_LAST!"} response = self.lg.handle_request( method="PATCH", path=f"/responses/{responseId}", headers={ "authorization": "auth", "Content-Type": "application/json" }, body=json.dumps(body), ) expected_data = copy.deepcopy(ONE_FORMDATA) set_(expected_data, "contact_name.last", "NEW_LAST!") self.assertEqual(response["statusCode"], 200, response) body = json.loads(response["body"]) self.assertEqual(body["res"]["response"]["value"], expected_data) self.assertEqual( [remove_date(i) for i in body["res"]["response"]["update_trail"]], [ { "old": { "contact_name": { "first": "Ashwin", "last": "Ash" }, "address": { "line1": "123", "line2": "asdad", "city": "Atlanta", "state": "GA", "zipcode": "30022", }, "email": "*****@*****.**", "phone": "1231231233", "amount": 0.5, "subscribe": True, }, "new": { "contact_name": { "first": "Ashwin", "last": "Ash" }, "address": { "line1": "123", "line2": "asdad", "city": "Atlanta", "state": "GA", "zipcode": "30022", }, "email": "*****@*****.**", "phone": "1231231233", "amount": 0.5, "subscribe": True, }, "update_type": "update", "_cls": "chalicelib.models.UpdateTrailItem", }, { "path": "contact_name.last", "user": "******", "old_value": "Ash", "new_value": "NEW_LAST!", "response_base_path": "value", "_cls": "chalicelib.models.UpdateTrailItem", }, ], )
def substitute(d, aliases): for alias, key in aliases.items(): if has(d, key): set_(d, alias, get(d, key)) unset(d, key)
def process_item(self, item): docs, material_dict = item grouped = group_by_material_id(material_dict, docs, 'input_structure') formula = docs[0]['pretty_formula'] if not grouped: formula = Structure.from_dict(list( material_dict.values())[0]).composition.reduced_formula logger.debug("No material match for {}".format(formula)) # For now just do the most recent one that's not failed # TODO: better sorting of docs all_docs = [] for task_id, elastic_docs in grouped.items(): elastic_docs = sorted(elastic_docs, key=lambda x: (x['order'], x['state'], x['completed_at'])) grouped_by_order = { k: list(v) for k, v in groupby(elastic_docs, key=lambda x: x['order']) } soec_docs = grouped_by_order.get(2) toec_docs = grouped_by_order.get(3) if soec_docs: final_doc = soec_docs[-1] else: final_doc = toec_docs[-1] structure = Structure.from_dict(final_doc['optimized_structure']) formula = structure.composition.reduced_formula elements = [s.symbol for s in structure.composition.elements] chemsys = '-'.join(elements) # Issue warning if relaxed structure differs warnings = final_doc.get('warnings') or [] opt = Structure.from_dict(final_doc['optimized_structure']) init = Structure.from_dict(final_doc['input_structure']) # TODO: are these the right params? if not StructureMatcher().fit(init, opt): warnings.append("Inequivalent optimization structure") material_mag = CollinearMagneticStructureAnalyzer( opt).ordering.value material_mag = mag_types[material_mag] if final_doc['magnetic_type'] != material_mag: warnings.append("Elastic magnetic phase is {}".format( final_doc['magnetic_type'])) warnings = warnings or None # Filter for failure and warnings k_vrh = final_doc['k_vrh'] if k_vrh < 0 or k_vrh > 600: state = 'failed' elif warnings is not None: state = 'warning' else: state = 'successful' final_doc.update({"warnings": warnings}) elastic_summary = { 'task_id': task_id, 'all_elastic_fits': elastic_docs, 'elasticity': final_doc, 'spacegroup': init.get_space_group_info()[0], 'magnetic_type': final_doc['magnetic_type'], 'pretty_formula': formula, 'chemsys': chemsys, 'elements': elements, 'last_updated': self.elasticity.lu_field, 'state': state } if toec_docs: # TODO: this should be a bit more refined final_toec_doc = deepcopy(toec_docs[-1]) et_exp = ElasticTensorExpansion.from_voigt( final_toec_doc['elastic_tensor_expansion']) symbol_dict = et_exp[1].zeroed(1e-2).get_symbol_dict() final_toec_doc.update({"symbol_dict": symbol_dict}) set_(elastic_summary, "elasticity.third_order", final_toec_doc) all_docs.append(jsanitize(elastic_summary)) # elastic_summary.update(final_doc) return all_docs
def make_mat(self, task_group): """ Converts a group of tasks into one material """ # Convert the task to properties and flatten all_props = list( chain.from_iterable([self.task_to_prop_list(t) for t in task_group]) ) mat_id = find_mat_id(all_props) # Sort and group based on property sorted_props = sorted(all_props, key=lambda prop: prop["materials_key"]) grouped_props = groupby(sorted_props, key=lambda prop: prop["materials_key"]) # Choose the best prop for each materials key: highest quality score and lowest energy calculation best_props = [find_best_prop(props) for _, props in grouped_props] # Add in the provenance for the properties origins = [ { k: prop[k] for k in ["materials_key", "task_type", "task_id", "last_updated"] } for prop in best_props if prop.get("track", False) ] # Store any bad props invalid_props = [ prop["materials_key"] for prop in best_props if not prop["is_valid"] ] # Store all the task_ids task_ids = list(set([t["task_id"] for t in task_group])) deprecated_tasks = list( set([t["task_id"] for t in task_group if not t.get("is_valid", True)]) ) # Store task_types task_types = {t["task_id"]: t["task_type"] for t in all_props} # Store sandboxes sandboxes = list(set(chain.from_iterable([k["sbxn"] for k in best_props]))) mat = { self.materials.lu_field: max([prop["last_updated"] for prop in all_props]), "created_at": min([prop["last_updated"] for prop in all_props]), "task_ids": task_ids, "deprecated_tasks": deprecated_tasks, self.materials.key: mat_id, "origins": origins, "task_types": task_types, "invalid_props": invalid_props, "_sbxn": sandboxes, } for prop in best_props: set_(mat, prop["materials_key"], prop["value"]) return mat
def make_mat(self, task_group): """ Converts a group of tasks into one material """ # Convert the task to properties and flatten all_props = list(chain.from_iterable([self.task_to_prop_list(t) for t in task_group])) # Store task_id of first structure as material task_id structure_task_ids = list( sorted( [ prop["task_id"] for prop in all_props if prop["materials_key"] == "structure" and "Structure Optimization" in prop["task_type"] ], key=ID_to_int)) # If we don't have a structure optimization then just return no material if len(structure_task_ids) == 0 and self.require_structure_opt: return None # Sort and group based on materials key sorted_props = sorted(all_props, key=lambda x: x['materials_key']) grouped_props = groupby(sorted_props, lambda x: x['materials_key']) # Choose the best prop for each materials key: highest quality score and lowest energy best_props = [] for _, props in grouped_props: # Sort for highest quality score and lowest energy sorted_props = sorted(props, key=lambda x: (x['quality_score'], -1.0 * x["energy"]), reverse=True) if sorted_props[0].get("aggregate", False): vals = [prop["value"] for prop in sorted_props] prop = sorted_props[0] prop["value"] = vals # Can't track an aggregated property prop["track"] = False best_props.append(prop) else: best_props.append(sorted_props[0]) # Add in the provenance for the properties origins = [{k: prop[k] for k in ["materials_key", "task_type", "task_id", "last_updated"]} for prop in best_props if prop.get("track", False)] # Store all the task_ids task_ids = list(sorted([t["task_id"] for t in task_group], key=ID_to_int)) # Store task_types task_types = {t["task_id"]: t["task_type"] for t in all_props} mat = { self.materials.lu_field: max([prop["last_updated"] for prop in all_props]), "task_ids": task_ids, self.materials.key: structure_task_ids[0], "origins": origins, "task_types": task_types } for prop in best_props: set_(mat, prop["materials_key"], prop["value"]) # Add metadata back into document if "structure" in mat: structure = Structure.from_dict(mat["structure"]) mat.update(structure_metadata(structure)) return mat
def old_style_mat(new_mat): mat = {} for mp, new_key in mp_conversion_dict.items(): if has(new_mat, new_key): set_(mat, mp, get(new_mat, new_key)) mat["is_orderd"] = True mat["is_compatible"] = True struc = Structure.from_dict(mat["structure"]) mat["oxide_type"] = oxide_type(struc) mat["reduced_cell_formula"] = struc.composition.as_dict() mat["full_formula"] = "".join(struc.formula.split()) vals = sorted(mat["reduced_cell_formula"].values()) mat["anonymous_formula"] = { string.ascii_uppercase[i]: float(vals[i]) for i in range(len(vals)) } set_(mat, "original_task_id", get(new_mat, "material_id")) set_(mat, "ntask_ids", len(get(new_mat, "task_ids"))) set_(mat, "input.incar", get(new_mat, "inputs.structure_optimization.incar")) set_(mat, "input.kpoints", get(new_mat, "inputs.structure_optimization.kpoints")) set_(mat, "encut", get(new_mat, "inputs.structure_optimization.incar.ENCUT")) mat["pseudo_potential"] = { "pot_type": "paw", "labels": get(new_mat, "input.structure_optimization.potcar.symbols"), "functional": get(new_mat, "input.structure_optimization.potcar.functional") } return mat