db.authenticate(y['username'], y['password'])
snldb = SNLMongoAdapter.from_file(snl_f)
prev_ids = [] # MPS ids that we already took care of
print 'INITIALIZING'
if RESET:
snldb._reset()
time.sleep(10) # makes me sleep better at night
else:
for mps in snldb.snl.find({}, {"about._materialsproject.deprecated.mps_ids": 1}):
prev_ids.extend(mps['about']['_materialsproject']['deprecated']['mps_ids'])
print 'PROCESSING'
for mps in db.mps.find(timeout=False):
try:
if not mps['mps_id'] in prev_ids:
snl = mps_dict_to_snl(mps)
if snl:
snldb.add_snl(snl)
else:
print 'SKIPPING', mps['mps_id']
except:
traceback.print_exc()
print 'ERROR - mps id:', mps['mps_id']
print 'DONE'
def process_fw(self, old_task, d):
# AJ - this whole section is different
sma = SNLMongoAdapter.auto_load()
d['old_engine'] = old_task.get('engine')
if 'fw_id' in old_task:
d['old_fw_id'] = old_task['fw_id']
d['fw_id'] = None
d['task_type'] = 'GGA+U optimize structure (2x)' if old_task[
'is_hubbard'] else 'GGA optimize structure (2x)'
d['submission_id'] = None
d['vaspinputset_name'] = None
snl_d = sma.snl.find_one(
{'about._materialsproject.deprecated.mps_ids': old_task['mps_id']})
if old_task.get('mps_id', -1) > 0 and snl_d:
# grab the SNL from the SNL db
del snl_d['_id']
d['snl'] = snl_d
d['snlgroup_id'] = sma.snlgroups.find_one(
{'all_snl_ids': d['snl']['snl_id']},
{'snlgroup_id': 1})['snlgroup_id']
elif 'mps' in old_task and old_task['mps']:
snl = mps_dict_to_snl(old_task['mps'])
mpsnl, snlgroup_id = sma.add_snl(snl)
d['snl'] = mpsnl.as_dict()
d['snlgroup_id'] = snlgroup_id
else:
s = Structure.from_dict(old_task['input']['crystal'])
snl = StructureNL(s,
'Anubhav Jain <*****@*****.**>',
remarks=['origin unknown'])
mpsnl, snlgroup_id = sma.add_snl(snl)
d['snl'] = mpsnl.as_dict()
d['snlgroup_id'] = snlgroup_id
if 'optimize structure' in d['task_type'] and 'output' in d:
# create a new SNL based on optimized structure
new_s = Structure.from_dict(d['output']['crystal'])
old_snl = StructureNL.from_dict(d['snl'])
history = old_snl.history
history.append({
'name': 'Materials Project structure optimization',
'url': 'http://www.materialsproject.org',
'description': {
'task_type': d['task_type'],
'fw_id': d['fw_id'],
'task_id': d['task_id']
}
})
new_snl = StructureNL(new_s, old_snl.authors, old_snl.projects,
old_snl.references, old_snl.remarks,
old_snl.data, history)
# add snl
mpsnl, snlgroup_id = sma.add_snl(new_snl,
snlgroup_guess=d['snlgroup_id'])
d['snl_final'] = mpsnl.as_dict()
d['snlgroup_id_final'] = snlgroup_id
d['snlgroup_changed'] = (d['snlgroup_id'] !=
d['snlgroup_id_final'])
# custom processing for detecting errors
dir_name = old_task['dir_name']
new_style = os.path.exists(os.path.join(dir_name, 'FW.json'))
vasp_signals = {}
critical_errors = [
"INPUTS_DONT_EXIST", "OUTPUTS_DONT_EXIST", "INCOHERENT_POTCARS",
"VASP_HASNT_STARTED", "VASP_HASNT_COMPLETED", "CHARGE_UNCONVERGED",
"NETWORK_QUIESCED", "HARD_KILLED", "WALLTIME_EXCEEDED",
"ATOMS_TOO_CLOSE", "DISK_SPACE_EXCEEDED"
]
last_relax_dir = dir_name
if not new_style:
# get the last relaxation dir
# the order is relax2, current dir, then relax1. This is because
# after completing relax1, the job happens in the current dir.
# Finally, it gets moved to relax2.
# There are some weird cases where both the current dir and relax2
# contain data. The relax2 is good, but the current dir is bad.
if is_valid_vasp_dir(os.path.join(dir_name, "relax2")):
last_relax_dir = os.path.join(dir_name, "relax2")
elif is_valid_vasp_dir(dir_name):
pass
elif is_valid_vasp_dir(os.path.join(dir_name, "relax1")):
last_relax_dir = os.path.join(dir_name, "relax1")
vasp_signals['last_relax_dir'] = last_relax_dir
## see what error signals are present
print "getting signals for dir :{}".format(last_relax_dir)
sl = SignalDetectorList()
sl.append(VASPInputsExistSignal())
sl.append(VASPOutputsExistSignal())
sl.append(VASPOutSignal())
sl.append(HitAMemberSignal())
sl.append(SegFaultSignal())
sl.append(VASPStartedCompletedSignal())
signals = sl.detect_all(last_relax_dir)
signals = signals.union(WallTimeSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(WallTimeSignal().detect(root_dir))
signals = signals.union(DiskSpaceExceededSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(DiskSpaceExceededSignal().detect(root_dir))
signals = list(signals)
critical_signals = [val for val in signals if val in critical_errors]
vasp_signals['signals'] = signals
vasp_signals['critical_signals'] = critical_signals
vasp_signals['num_signals'] = len(signals)
vasp_signals['num_critical'] = len(critical_signals)
if len(critical_signals) > 0 and d['state'] == "successful":
d["state"] = "error"
d['analysis'] = d.get('analysis', {})
d['analysis']['errors_MP'] = vasp_signals
d['run_tags'] = ['PBE']
d['run_tags'].extend(d['pseudo_potential']['labels'])
d['run_tags'].extend(
[e + "=" + str(d['hubbards'].get(e, 0)) for e in d['elements']])
def process_fw(self, old_task, d):
# AJ - this whole section is different
sma = SNLMongoAdapter.auto_load()
d['old_engine'] = old_task.get('engine')
if 'fw_id' in old_task:
d['old_fw_id'] = old_task['fw_id']
d['fw_id'] = None
d['task_type'] = 'GGA+U optimize structure (2x)' if old_task[
'is_hubbard'] else 'GGA optimize structure (2x)'
d['submission_id'] = None
d['vaspinputset_name'] = None
snl_d = sma.snl.find_one({'about._materialsproject.deprecated.mps_ids': old_task['mps_id']})
if old_task.get('mps_id', -1) > 0 and snl_d:
# grab the SNL from the SNL db
del snl_d['_id']
d['snl'] = snl_d
d['snlgroup_id'] = sma.snlgroups.find_one({'all_snl_ids': d['snl']['snl_id']}, {'snlgroup_id': 1})['snlgroup_id']
elif 'mps' in old_task and old_task['mps']:
snl = mps_dict_to_snl(old_task['mps'])
mpsnl, snlgroup_id = sma.add_snl(snl)
d['snl'] = mpsnl.as_dict()
d['snlgroup_id'] = snlgroup_id
else:
s = Structure.from_dict(old_task['input']['crystal'])
snl = StructureNL(s, 'Anubhav Jain <*****@*****.**>', remarks=['origin unknown'])
mpsnl, snlgroup_id = sma.add_snl(snl)
d['snl'] = mpsnl.as_dict()
d['snlgroup_id'] = snlgroup_id
if 'optimize structure' in d['task_type'] and 'output' in d:
# create a new SNL based on optimized structure
new_s = Structure.from_dict(d['output']['crystal'])
old_snl = StructureNL.from_dict(d['snl'])
history = old_snl.history
history.append(
{'name': 'Materials Project structure optimization',
'url': 'http://www.materialsproject.org',
'description': {'task_type': d['task_type'],
'fw_id': d['fw_id'],
'task_id': d['task_id']}})
new_snl = StructureNL(new_s, old_snl.authors, old_snl.projects,
old_snl.references, old_snl.remarks,
old_snl.data, history)
# add snl
mpsnl, snlgroup_id = sma.add_snl(new_snl, snlgroup_guess=d['snlgroup_id'])
d['snl_final'] = mpsnl.as_dict()
d['snlgroup_id_final'] = snlgroup_id
d['snlgroup_changed'] = (d['snlgroup_id'] !=
d['snlgroup_id_final'])
# custom processing for detecting errors
dir_name = old_task['dir_name']
new_style = os.path.exists(os.path.join(dir_name, 'FW.json'))
vasp_signals = {}
critical_errors = ["INPUTS_DONT_EXIST",
"OUTPUTS_DONT_EXIST", "INCOHERENT_POTCARS",
"VASP_HASNT_STARTED", "VASP_HASNT_COMPLETED",
"CHARGE_UNCONVERGED", "NETWORK_QUIESCED",
"HARD_KILLED", "WALLTIME_EXCEEDED",
"ATOMS_TOO_CLOSE", "DISK_SPACE_EXCEEDED"]
last_relax_dir = dir_name
if not new_style:
# get the last relaxation dir
# the order is relax2, current dir, then relax1. This is because
# after completing relax1, the job happens in the current dir.
# Finally, it gets moved to relax2.
# There are some weird cases where both the current dir and relax2
# contain data. The relax2 is good, but the current dir is bad.
if is_valid_vasp_dir(os.path.join(dir_name, "relax2")):
last_relax_dir = os.path.join(dir_name, "relax2")
elif is_valid_vasp_dir(dir_name):
pass
elif is_valid_vasp_dir(os.path.join(dir_name, "relax1")):
last_relax_dir = os.path.join(dir_name, "relax1")
vasp_signals['last_relax_dir'] = last_relax_dir
## see what error signals are present
print "getting signals for dir :{}".format(last_relax_dir)
sl = SignalDetectorList()
sl.append(VASPInputsExistSignal())
sl.append(VASPOutputsExistSignal())
sl.append(VASPOutSignal())
sl.append(HitAMemberSignal())
sl.append(SegFaultSignal())
sl.append(VASPStartedCompletedSignal())
signals = sl.detect_all(last_relax_dir)
signals = signals.union(WallTimeSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(WallTimeSignal().detect(root_dir))
signals = signals.union(DiskSpaceExceededSignal().detect(dir_name))
if not new_style:
root_dir = os.path.dirname(dir_name) # one level above dir_name
signals = signals.union(DiskSpaceExceededSignal().detect(root_dir))
signals = list(signals)
critical_signals = [val for val in signals if val in critical_errors]
vasp_signals['signals'] = signals
vasp_signals['critical_signals'] = critical_signals
vasp_signals['num_signals'] = len(signals)
vasp_signals['num_critical'] = len(critical_signals)
if len(critical_signals) > 0 and d['state'] == "successful":
d["state"] = "error"
d['analysis'] = d.get('analysis', {})
d['analysis']['errors_MP'] = vasp_signals
d['run_tags'] = ['PBE']
d['run_tags'].extend(d['pseudo_potential']['labels'])
d['run_tags'].extend([e+"="+str(d['hubbards'].get(e, 0)) for e in d['elements']])
snldb = SNLMongoAdapter.from_file(snl_f)
prev_ids = [] # MPS ids that we already took care of
print 'INITIALIZING'
if RESET:
snldb._reset()
time.sleep(10) # makes me sleep better at night
else:
for mps in snldb.snl.find(
{}, {"about._materialsproject.deprecated.mps_ids": 1}):
prev_ids.extend(
mps['about']['_materialsproject']['deprecated']['mps_ids'])
print 'PROCESSING'
for mps in db.mps.find(timeout=False):
try:
if not mps['mps_id'] in prev_ids:
snl = mps_dict_to_snl(mps)
if snl:
snldb.add_snl(snl)
else:
print 'SKIPPING', mps['mps_id']
except:
traceback.print_exc()
print 'ERROR - mps id:', mps['mps_id']
print 'DONE'
