db = mc[y['db']]
db.authenticate(y['username'], y['password'])
snldb = SNLMongoAdapter.from_file(snl_f)
prev_ids = [] # MPS ids that we already took care of
print 'INITIALIZING'
if RESET:
snldb._reset()
time.sleep(10) # makes me sleep better at night
else:
for mps in snldb.snl.find({}, {"about._materialsproject.deprecated.mps_ids": 1}):
prev_ids.extend(mps['about']['_materialsproject']['deprecated']['mps_ids'])
print 'PROCESSING'
for mps in db.mps.find(timeout=False):
try:
if not mps['mps_id'] in prev_ids:
snl = mps_dict_to_snl(mps)
if snl:
snldb.add_snl(snl)
else:
print 'SKIPPING', mps['mps_id']
except:
traceback.print_exc()
print 'ERROR - mps id:', mps['mps_id']
print 'DONE'
def process_fw(self, old_task, d):
# AJ - this whole section is different
sma = SNLMongoAdapter.auto_load()
d['old_engine'] = old_task.get('engine')
if 'fw_id' in old_task:
d['old_fw_id'] = old_task['fw_id']
d['fw_id'] = None
d['task_type'] = 'GGA+U optimize structure (2x)' if old_task[
'is_hubbard'] else 'GGA optimize structure (2x)'
d['submission_id'] = None
d['vaspinputset_name'] = None
snl_d = sma.snl.find_one({'about._materialsproject.deprecated.mps_ids': old_task['mps_id']})
if old_task.get('mps_id', -1) > 0 and snl_d:
# grab the SNL from the SNL db
del snl_d['_id']
d['snl'] = snl_d
d['snlgroup_id'] = sma.snlgroups.find_one({'all_snl_ids': d['snl']['snl_id']}, {'snlgroup_id': 1})['snlgroup_id']
elif 'mps' in old_task and old_task['mps']:
snl = mps_dict_to_snl(old_task['mps'])
mpsnl, snlgroup_id = sma.add_snl(snl)
d['snl'] = mpsnl.to_dict
d['snlgroup_id'] = snlgroup_id
else:
s = Structure.from_dict(old_task['input']['crystal'])
snl = StructureNL(s, 'Anubhav Jain <*****@*****.**>', remarks=['origin unknown'])
mpsnl, snlgroup_id = sma.add_snl(snl)
d['snl'] = mpsnl.to_dict
d['snlgroup_id'] = snlgroup_id
if 'optimize structure' in d['task_type'] and 'output' in d:
# create a new SNL based on optimized structure
new_s = Structure.from_dict(d['output']['crystal'])
old_snl = StructureNL.from_dict(d['snl'])
history = old_snl.history
history.append(
{'name': 'Materials Project structure optimization',
'url': 'http://www.materialsproject.org',
'description': {'task_type': d['task_type'],
'fw_id': d['fw_id'],
'task_id': d['task_id']}})
new_snl = StructureNL(new_s, old_snl.authors, old_snl.projects,
old_snl.references, old_snl.remarks,
old_snl.data, history)
# add snl
mpsnl, snlgroup_id = sma.add_snl(new_snl, snlgroup_guess=d['snlgroup_id'])
d['snl_final'] = mpsnl.to_dict
d['snlgroup_id_final'] = snlgroup_id
d['snlgroup_changed'] = (d['snlgroup_id'] !=
d['snlgroup_id_final'])
# custom processing for detecting errors
dir_name = old_task['dir_name']
new_style = os.path.exists(os.path.join(dir_name, 'FW.json'))
vasp_signals = {}
critical_errors = ["INPUTS_DONT_EXIST",
"OUTPUTS_DONT_EXIST", "INCOHERENT_POTCARS",
"VASP_HASNT_STARTED", "VASP_HASNT_COMPLETED",
"CHARGE_UNCONVERGED", "NETWORK_QUIESCED",
"HARD_KILLED", "WALLTIME_EXCEEDED",
"ATOMS_TOO_CLOSE", "DISK_SPACE_EXCEEDED"]
last_relax_dir = dir_name
if not new_style:
# get the last relaxation dir
# the order is relax2, current dir, then relax1. This is because
# after completing relax1, the job happens in the current dir.
# Finally, it gets moved to relax2.
# There are some weird cases where both the current dir and relax2
# contain data. The relax2 is good, but the current dir is bad.
if is_valid_vasp_dir(os.path.join(dir_name, "relax2")):
last_relax_dir = os.path.join(dir_name, "relax2")
elif is_valid_vasp_dir(dir_name):
pass
elif is_valid_vasp_dir(os.path.join(dir_name, "relax1")):
last_relax_dir = os.path.join(dir_name, "relax1")
vasp_signals['last_relax_dir'] = last_relax_dir
## see what error signals are present
print "getting signals for dir :{}".format(last_relax_dir)
sl = SignalDetectorList()
sl.append(VASPInputsExistSignal())
sl.append(VASPOutputsExistSignal())
sl.append(VASPOutSignal())
sl.append(HitAMemberSignal())
sl.append(SegFaultSignal())
sl.append(VASPStartedCompletedSignal())
signals = sl.detect_all(last_relax_dir)
signals = signals.union(WallTimeSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(WallTimeSignal().detect(root_dir))
signals = signals.union(DiskSpaceExceededSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(DiskSpaceExceededSignal().detect(root_dir))
signals = list(signals)
critical_signals = [val for val in signals if val in critical_errors]
vasp_signals['signals'] = signals
vasp_signals['critical_signals'] = critical_signals
vasp_signals['num_signals'] = len(signals)
vasp_signals['num_critical'] = len(critical_signals)
if len(critical_signals) > 0 and d['state'] == "successful":
d["state"] = "error"
d['analysis'] = d.get('analysis', {})
d['analysis']['errors_MP'] = vasp_signals
d['run_tags'] = ['PBE']
d['run_tags'].extend(d['pseudo_potential']['labels'])
d['run_tags'].extend([e+"="+str(d['hubbards'].get(e, 0)) for e in d['elements']])
def process_fw(self, old_task, d):
# AJ - this whole section is different
sma = SNLMongoAdapter.auto_load()
d["old_engine"] = old_task.get("engine")
if "fw_id" in old_task:
d["old_fw_id"] = old_task["fw_id"]
d["fw_id"] = None
d["task_type"] = "GGA+U optimize structure (2x)" if old_task["is_hubbard"] else "GGA optimize structure (2x)"
d["submission_id"] = None
d["vaspinputset_name"] = None
snl_d = sma.snl.find_one({"about._materialsproject.deprecated.mps_ids": old_task["mps_id"]})
if old_task.get("mps_id", -1) > 0 and snl_d:
# grab the SNL from the SNL db
del snl_d["_id"]
d["snl"] = snl_d
d["snlgroup_id"] = sma.snlgroups.find_one({"all_snl_ids": d["snl"]["snl_id"]}, {"snlgroup_id": 1})[
"snlgroup_id"
]
elif "mps" in old_task and old_task["mps"]:
snl = mps_dict_to_snl(old_task["mps"])
mpsnl, snlgroup_id = sma.add_snl(snl)
d["snl"] = mpsnl.as_dict()
d["snlgroup_id"] = snlgroup_id
else:
s = Structure.from_dict(old_task["input"]["crystal"])
snl = StructureNL(s, "Anubhav Jain <*****@*****.**>", remarks=["origin unknown"])
mpsnl, snlgroup_id = sma.add_snl(snl)
d["snl"] = mpsnl.as_dict()
d["snlgroup_id"] = snlgroup_id
if "optimize structure" in d["task_type"] and "output" in d:
# create a new SNL based on optimized structure
new_s = Structure.from_dict(d["output"]["crystal"])
old_snl = StructureNL.from_dict(d["snl"])
history = old_snl.history
history.append(
{
"name": "Materials Project structure optimization",
"url": "http://www.materialsproject.org",
"description": {"task_type": d["task_type"], "fw_id": d["fw_id"], "task_id": d["task_id"]},
}
)
new_snl = StructureNL(
new_s, old_snl.authors, old_snl.projects, old_snl.references, old_snl.remarks, old_snl.data, history
)
# add snl
mpsnl, snlgroup_id = sma.add_snl(new_snl, snlgroup_guess=d["snlgroup_id"])
d["snl_final"] = mpsnl.as_dict()
d["snlgroup_id_final"] = snlgroup_id
d["snlgroup_changed"] = d["snlgroup_id"] != d["snlgroup_id_final"]
# custom processing for detecting errors
dir_name = old_task["dir_name"]
new_style = os.path.exists(os.path.join(dir_name, "FW.json"))
vasp_signals = {}
critical_errors = [
"INPUTS_DONT_EXIST",
"OUTPUTS_DONT_EXIST",
"INCOHERENT_POTCARS",
"VASP_HASNT_STARTED",
"VASP_HASNT_COMPLETED",
"CHARGE_UNCONVERGED",
"NETWORK_QUIESCED",
"HARD_KILLED",
"WALLTIME_EXCEEDED",
"ATOMS_TOO_CLOSE",
"DISK_SPACE_EXCEEDED",
]
last_relax_dir = dir_name
if not new_style:
# get the last relaxation dir
# the order is relax2, current dir, then relax1. This is because
# after completing relax1, the job happens in the current dir.
# Finally, it gets moved to relax2.
# There are some weird cases where both the current dir and relax2
# contain data. The relax2 is good, but the current dir is bad.
if is_valid_vasp_dir(os.path.join(dir_name, "relax2")):
last_relax_dir = os.path.join(dir_name, "relax2")
elif is_valid_vasp_dir(dir_name):
pass
elif is_valid_vasp_dir(os.path.join(dir_name, "relax1")):
last_relax_dir = os.path.join(dir_name, "relax1")
vasp_signals["last_relax_dir"] = last_relax_dir
## see what error signals are present
print "getting signals for dir :{}".format(last_relax_dir)
sl = SignalDetectorList()
sl.append(VASPInputsExistSignal())
sl.append(VASPOutputsExistSignal())
sl.append(VASPOutSignal())
sl.append(HitAMemberSignal())
sl.append(SegFaultSignal())
sl.append(VASPStartedCompletedSignal())
signals = sl.detect_all(last_relax_dir)
signals = signals.union(WallTimeSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(WallTimeSignal().detect(root_dir))
signals = signals.union(DiskSpaceExceededSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(DiskSpaceExceededSignal().detect(root_dir))
signals = list(signals)
critical_signals = [val for val in signals if val in critical_errors]
vasp_signals["signals"] = signals
vasp_signals["critical_signals"] = critical_signals
vasp_signals["num_signals"] = len(signals)
vasp_signals["num_critical"] = len(critical_signals)
if len(critical_signals) > 0 and d["state"] == "successful":
d["state"] = "error"
d["analysis"] = d.get("analysis", {})
d["analysis"]["errors_MP"] = vasp_signals
d["run_tags"] = ["PBE"]
d["run_tags"].extend(d["pseudo_potential"]["labels"])
d["run_tags"].extend([e + "=" + str(d["hubbards"].get(e, 0)) for e in d["elements"]])
