def launchpad(launchpad_file=None, lpad_name="base"):
"""
Script to set up the configuration of the launchpad for accessing the workflow
server.
Args:
launchpad_file (str): my_launchpad.yaml file from which to load the
mongoDB database details.
lpad_name (str): Name to give to the database in the configuration setup.
Returns:
None
"""
if launchpad_file:
lpad = LaunchPad.from_file(launchpad_file)
else:
host = input("Please provide the server host: ")
port = int(input("Please provide the port number: "))
name = input("Please provide the server name: ")
username = input("Please provide your username: "******"Please provide your password: "******"admin")
# Test the launchpad
print("\nAttempting connection to mongoDB database...")
_ = lpad.get_fw_ids()
print("Connection successful!\n")
config_lpad_file = os.path.join(os.path.expanduser("~"),
".workflow_config", "launchpad",
lpad_name + "_launchpad.yaml")
try:
os.makedirs(
os.path.join(os.path.expanduser("~"), ".workflow_config",
"launchpad"))
except FileExistsError:
pass
lpad.to_file(config_lpad_file)
print("Launchpad file written to " + config_lpad_file + "\n")
#!/usr/bin/env python
from fireworks import LaunchPad, Firework, Workflow, PyTask
from ase.io import read
from qescripts.fwio import atoms_to_encode, encode_to_atoms
from ase.db import connect
from ase.visualize import view
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
ids = launchpad.get_fw_ids(query={'state': 'READY'})
for i in ids:
if i > 26470:
print('Processing: {0}'.format(i))
launch = launchpad.get_fw_by_id(i)
atoms = encode_to_atoms(launch.spec['_tasks'][0]['args'][0])[0]
try:
metal = launch.name['metal']
print(metal)
if '2' in metal:
a, b, _ = metal.split('2')
elif '3' in metal:
a, b = metal.split('3')
else:
#!/usr/bin/env python
from fireworks import LaunchPad, Firework, Workflow, PyTask
from ase.io import read
from catkit.flow.fwio import atoms_to_encode, encode_to_atoms
from ase.db import connect
from ase.visualize import view
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
IDS = launchpad.get_fw_ids(query={'state': 'COMPLETED'})
IDS = [i for i in IDS if i > 51775]
#IDS = [31032]
l_dir = []
for i in IDS:
print('Processing: {0}'.format(i))
launch = launchpad.get_fw_dict_by_id(i)
l_dir += [launch['launches'][-1]['launch_dir']]
print(sum([1 for _ in l_dir if 'nfs' in _]))
print(len(l_dir))
#LAST UPDATED: 05-22-2018
from fireworks import LaunchPad, Firework, Workflow, PyTask
from qescripts.fwio import atoms_to_encode, encode_to_atoms
import numpy as np
import re
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
ids = sorted(launchpad.get_fw_ids(query={'state': 'COMPLETED'}))
ids = [603]
culprit = []
halar_po = []
slac_id = []
for i in ids:
print('Processing: {0}'.format(i))
launch = launchpad.get_fw_dict_by_id(i)
launchdir = launch['launches'][-1]['launch_dir']
cluster = launch['launches'][-1]['fworker']['name']
print(cluster)
if cluster == 'SLAC':
print('Dis calculation was done in SLAC cluster!')
slac_id += [i]
#!/usr/bin/env python
from fireworks import LaunchPad, Firework, Workflow, PyTask
from ase.io import read
from qescripts.fwio import atoms_to_encode, encode_to_atoms
from ase.db import connect
from ase.visualize import view
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
ids = launchpad.get_fw_ids(query={'state': 'RUNNING'})
for i in ids:
print('Processing: {0}'.format(i))
launch = launchpad.get_fw_by_id(i)
atoms = encode_to_atoms(launch.spec['_tasks'][0]['args'][0])[0]
try:
metal = launch.name['calc']['metal']
adsorbate = launch.name['calc']['adsorbate']
site = launch.name['calc']['site']
except KeyError:
continue
pol_metal = ['Fe', 'Co', 'Ni', 'Mn']
#!/usr/bin/env python
from fireworks import LaunchPad, Firework, Workflow, PyTask
from ase.io import read
from qescripts.fwio import atoms_to_encode, encode_to_atoms
from ase.db import connect
from ase.visualize import view
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
ids = launchpad.get_fw_ids(query={'state': 'DEFUSED'})
adsorbate = ['C', 'O', 'H', 'S', 'N']
ten_metal = ['Pt', 'Pd', 'Rh', 'Ru', 'Re', 'Au', 'Bi', 'Co', 'Ag', 'Cu']
for i in ids:
print('Processing: {0}'.format(i))
launch = launchpad.get_fw_by_id(i)
atoms = encode_to_atoms(launch.spec['_tasks'][0]['args'][0])[0]
try:
metal = launch.name['calc']['symbol']
ads = launch.name['calc']['adsorbate']
except KeyError:
continue
class WFlowBuilder(object):
#This class builds and submits workflow(s) for a particular job submission
def __init__(self):
self.simManager = SimManager()
self.launchpad = LaunchPad()
self.ids = []
self.fws = []
self.last = 0
self.rerun = False
def addRunSteps(self, inpSim, userID, cores):
#builds the following workflow for running simulations
#creates directory for simulation using the sim name and uuid
#writes gkyl input file to the directory
#runs file in the directory
#if run is successful, prints 'Done', if not, prints 'Failed
#plots task
#userid --> for folder location
self.fws = []
print(inpSim)
self.last = len(self.launchpad.get_fw_ids())
# for fwork in self.launchpad.get_fw_ids():
# self.launchpad.delete_fws([fwork])
sim = Sim(inpSim)
# path = '/home/adaniel99/gkylsoft/sims/'+str(userID)+'/'+inpSim+'/'
path = '/home/dalex_99/gkylsoft/sims/' + str(
userID) + '/' + inpSim + '/'
n = 0
# for f in os.listdir('/home/adaniel99/gkylsoft/sims/'):
for f in os.listdir('/home/dalex_99/gkylsoft/sims/'):
if f == str(userID):
# for f in os.listdir('/home/adaniel99/gkylsoft/sims/'+str(userID)+'/'):
for f in os.listdir('/home/dalex_99/gkylsoft/sims/' +
str(userID) + '/'):
if f == inpSim:
n = n + 1
self.rerun = True
if n == 0:
desttask = ScriptTask.from_str('mkdir ' + path)
writetask = FileWriteTask({
'files_to_write': ([{
'filename': sim.name(),
'contents': sim.inpFile()
}]),
'dest':
path
})
runtask = ScriptTask.from_str(
'redis-cli PUBLISH ' + User(userID).name() +
'2 "Running Simulation"; mpiexec -n ' + cores + 'gkyl ' +
path + sim.name())
runFlag = ScriptTask.from_str('redis-cli PUBLISH ' +
User(userID).name() + '2' + ' Done')
deleteFail = ScriptTask.from_str('lpad defuse_fws -i ' +
str(6 + self.last))
flagFail = ScriptTask.from_str('redis-cli PUBLISH ' +
User(userID).name() + '2' +
' Failed')
self.ids.clear()
dest = Firework(desttask, name='Make Folder', fw_id=1 + self.last)
self.ids.append(1 + self.last)
write = Firework(writetask, name='Write', fw_id=2 + self.last)
self.ids.append(2 + self.last)
run = Firework(runtask, name='Run', fw_id=3 + self.last)
self.ids.append(3 + self.last)
flag1 = Firework(runFlag, name='done?', fw_id=4 + self.last)
self.ids.append(4 + self.last)
delfail = Firework(deleteFail,
name='remove fail flag',
fw_id=5 + self.last)
self.ids.append(5 + self.last)
failflag = Firework(flagFail,
name='fail flag',
fw_id=6 + self.last)
self.ids.append(6 + self.last)
self.fws.append(dest)
self.fws.append(write)
self.fws.append(run)
self.fws.append(flag1)
self.fws.append(delfail)
self.fws.append(failflag)
wf = Workflow(self.fws, {
dest: [write],
write: [run],
run: [flag1],
flag1: [delfail]
},
name='Running ' + sim.name())
self.launchpad.add_wf(wf)
if n == 1:
writetask = FileWriteTask({
'files_to_write': ([{
'filename': sim.name(),
'contents': sim.inpFile()
}]),
'dest':
path
})
runtask = ScriptTask.from_str(
'redis-cli PUBLISH ' + User(userID).name() +
'2 "Running Simulation"; mpiexec -n ' + cores + 'gkyl ' +
path + sim.name())
runFlag = ScriptTask.from_str('redis-cli PUBLISH ' +
User(userID).name() + '2' + ' Done')
deleteFail = ScriptTask.from_str('lpad defuse_fws -i ' +
str(5 + self.last))
flagFail = ScriptTask.from_str('redis-cli PUBLISH ' +
User(userID).name() + '2' +
' Failed')
self.ids.clear()
write = Firework(writetask, name='Write', fw_id=1 + self.last)
self.ids.append(1 + self.last)
run = Firework(runtask, name='Run', fw_id=2 + self.last)
self.ids.append(2 + self.last)
flag1 = Firework(runFlag, name='done?', fw_id=3 + self.last)
self.ids.append(3 + self.last)
delfail = Firework(deleteFail,
name='remove fail flag',
fw_id=4 + self.last)
self.ids.append(4 + self.last)
failflag = Firework(flagFail,
name='fail flag',
fw_id=5 + self.last)
self.ids.append(5 + self.last)
wf = Workflow([write, run, flag1, delfail, failflag], {
write: [run],
run: [flag1],
flag1: [delfail]
},
name='Running ' + sim.name())
self.launchpad.add_wf(wf)
def runAll(self):
for i in self.ids:
launch_rocket(self.launchpad, self.worker, fw_id=i)
def runParallel(self):
#test
#launch two jobs simultaneously (2, one on each core)
launch_multiprocess(self.launchpad, self.worker, 'INFO', 0, 2, 10)
def getFWs(self):
fws1 = []
for ids in self.ids:
fws1.append[self.launchpad.get_fw_by_id(ids)]
self.fws = fws1
return self.fws
def slurm_launch(self):
print("slurm_launch method")
if self.rerun == True:
print(self.launchpad.get_fw_ids())
print(self.rerun)
for i in self.ids:
#os.system('salloc --core-spec=1 --time=5 --partition=VME lpad rerun_fws -i '+str(i))
# os.system('salloc --core-spec=1 --time=5 --partition=VME rlaunch singleshot -f '+ str(i))
os.system(
'salloc --core-spec=1 --time=5 --partition=xps rlaunch singleshot -f '
+ str(i))
if self.rerun == False:
print(self.launchpad.get_fw_ids())
print(self.rerun)
for i in self.ids:
# os.system('salloc --core-spec=1 --time=5 --partition=VME rlaunch singleshot -f '+ str(i))
os.system(
'salloc --core-spec=1 --time=5 --partition=xps rlaunch singleshot -f '
+ str(i))
def simStates(self):
#gets current state (str) for each firework in the queue
#states: ('READY', 'WAITING', 'RUNNING', or 'COMPLETE')
state = []
for fWs in self.fws:
state.append(fWs.state())
return state
import numpy as np
import datetime
import sys
time = datetime.datetime(2018, 1, 1)
# Read credentials from a secure location
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
# Select the ID of the first completed calcualtion
IDS = launchpad.get_fw_ids(query={'state': 'COMPLETED'
}) #, 'time_end': {"$gt" : time}})
IDS = sorted(IDS)
a = 0
b = 0
FWID = []
for ID in IDS:
print(str(ID) + '\n')
launch = launchpad.get_fw_dict_by_id(ID)
ldir = launch['launches'][-1]['launch_dir'].encode('utf-8')
cluster = launch['launches'][-1]['fworker']['name'].encode('utf-8')
if cluster == "SLAC":
a += 1
print(ldir, cluster)
if os.path.exists(ldir):
from fireworks import LaunchPad, Firework, Workflow, PyTask
from catkit.flow.fwio import atoms_to_encode, encode_to_atoms
import numpy as np
import re
import os
from ase.constraints import FixAtoms
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
IDS = sorted(launchpad.get_fw_ids(query={"state": 'READY'}))
for i in IDS:
print('Processing: {0}'.format(i))
launch = launchpad.get_fw_by_id(i)
atoms = encode_to_atoms(launch.spec['_tasks'][0]['args'][0])[0]
atoms.info['convergence']['energy'] = 1e-4
atoms.info['convergence']['mixing'] = 0.2
atoms.info['fmax'] = 0.07
atoms.info['sigma'] = 0.25
atoms._calc = None
encoding = atoms_to_encode(atoms)
launchpad.update_spec([i], spec_document={'_tasks.0.args.0': encoding})
import json
import os
import numpy as np
import datetime
import sys
from catkit.flow.fwio import atoms_to_encode, encode_to_atoms
host = 'suncatls2.slac.stanford.edu'
username, name, password = netrc().authenticators(host)
launchpad = LaunchPad(host=host,
name=name,
username=username,
password=password)
IDS = launchpad.get_fw_ids(query={'state': 'FIZZLED'})
for i in IDS:
print('Processing: {0}'.format(i))
launch = launchpad.get_fw_dict_by_id(i)
encoding = launch['launches'][-1]
print(encoding)
atoms = encode_to_atoms(encoding)[-1]
atoms.info['calculator_parameters'] = {
'calculator_name': 'decaf.Espresso',
'calculation': 'relax',
'input_dft': 'BEEF-vdW',
'ecutwfc': 500,
'ecutrho': 5000,