def qadapter(qadapter_file=None, fworker_name="base"):
"""
Script to set up the configuration of the queueing system. Note that we store
the queue_adapter in the same configuration directory as the Fireworker,
i.e. fworker. This is in the assumption that each worker has one preferred
queueing system.
Args:
qadapter_file (str): my_qadapter_file.yaml from which to configure the
queue adapter.
fworker_name (str): Which fworker configuration to add the queue adapter
to. Note that there has to be a corresponding job_template.sh file for
the queue adapter to adjust before submitting it to the fireworker queue.
Returns:
None
"""
if qadapter_file:
queue_adapter = CommonAdapter.from_file(qadapter_file)
else:
logdir = input(
"Please provide the path to the directory where the log "
"should be stored. \n"
"(Note: if the directory does not exist, it will be created): ")
logdir = os.path.abspath(logdir)
if not os.path.exists(logdir):
os.makedirs(logdir)
queue_adapter = CommonAdapter.from_file(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"examples", "config", "fworker",
"example_qadapter.yaml"))
queue_adapter["logdir"] = logdir
print(
"\nNote: 'rocket_launch' has been set to an infinite rapidfire mode."
)
try:
os.makedirs(
os.path.join(os.path.expanduser("~"), ".workflow_config",
"fworker"))
except FileExistsError:
pass
config_q_file = os.path.join(os.path.expanduser("~"), ".workflow_config",
"fworker", fworker_name + "_qadapter.yaml")
template_file = os.path.join(os.path.expanduser("~"), ".workflow_config",
"fworker", fworker_name + "_job_template.sh")
if not os.path.exists(template_file):
print()
warn_message = "No corresponding template file found! Don't forget to use " \
"'vsc config jobscript' to add the template file of the '" \
+ fworker_name + "' fireworker."
warnings.warn(warn_message)
queue_adapter.template_file = template_file
queue_adapter.to_file(config_q_file)
print("\nQueue adapter file written to " + config_q_file)
def test_serialization_details(self):
# This detects a weird bug found in early version of serializers
pbs = CommonAdapter("PBS")
self.assertTrue(isinstance(pbs, CommonAdapter))
self.assertTrue(isinstance(self.get_data(pbs.to_dict()), CommonAdapter))
self.assertTrue(isinstance(load_object(pbs.to_dict()), CommonAdapter))
self.assertTrue(isinstance(self.get_data(pbs.to_dict()), CommonAdapter)) # repeated test on purpose!
def test_serialization_details(self):
# This detects a weird bug found in early version of serializers
pbs = CommonAdapter('PBS')
self.assertTrue(isinstance(pbs, CommonAdapter))
self.assertTrue(isinstance(self.get_data(pbs.to_dict()), CommonAdapter))
self.assertTrue(isinstance(load_object(pbs.to_dict()), CommonAdapter))
self.assertTrue(isinstance(self.get_data(pbs.to_dict()), CommonAdapter)) # repeated test on purpose!
def from_dict(cls, d):
qadapter = None
if d["qadapter"] is not None:
qadapter = CommonAdapter.from_dict(d["qadapter"])
return MPINTLammpsInput(MPINTLammps.from_dict(d["mplmp"]),
qadapter,
vis_logger=logging.getLogger(d["logger"]))
def load_config(config, name="base"):
"""
Load a LaunchPad, FWorker of QueueAdapter from the configuration files.
Args:
config (str): Type of configuration file to load. Either "launchpad", "fworker"
or "qadapter".
name (str): Name of the configuration. Defaults to "base".
Returns:
Either a LaunchPad, FWorker or QueuAdapter, depending on the "config" argument.
"""
try:
if config == "launchpad":
return LaunchPad.from_file(
os.path.join(os.path.expanduser("~"), ".workflow_config",
"launchpad", name + "_launchpad.yaml"))
if config == "fworker":
return FWorker.from_file(
os.path.join(os.path.expanduser("~"), ".workflow_config",
"fworker", name + "_fworker.yaml"))
if config == "qadapter":
return CommonAdapter.from_file(
os.path.join(os.path.expanduser("~"), ".workflow_config",
"fworker", name + "_qadapter.yaml"))
except FileNotFoundError:
raise FileNotFoundError(
"Did not find the corresponding configuration file in " +
os.path.join(os.path.expanduser("~"), ".workflow_config") +
". Use 'vsc config " + config + "' to set up the " + name +
" configuration for the " + config + ".")
def from_dict(cls, d):
qadapter = None
if d["qadapter"] is not None:
qadapter = CommonAdapter.from_dict(d["qadapter"])
return MPINTLammpsInput(
MPINTLammps.from_dict(d["mplmp"]),
qadapter,
vis_logger = logging.getLogger(d["logger"]))
def get_run_cmmnd(nnodes=1,
nprocs=16,
walltime='24:00:00',
job_bin=None,
mem='1000'):
d = {}
job_cmd = None
hostname = socket.gethostname()
# hipergator
if 'ufhpc' in hostname:
if job_bin is None:
job_bin = '/home/km468/Software/VASP/vasp.5.3.5/vasp'
else:
job_bin = job_bin
d = {
'type': 'PBS',
'params': {
'nnodes': str(nnodes),
'ppnode': str(int(nprocs / nnodes)),
'walltime': walltime,
'job_name': 'vasp_job',
'email': '*****@*****.**',
'notification_options': 'ae',
'pre_rocket': '#PBS -l pmem=' + str(mem) + 'mb',
'rocket_launch': 'mpirun ' + job_bin
}
}
# stampede
elif 'stampede' in hostname:
if job_bin is None:
job_bin = '/home1/01682/km468/Software/VASP/vasp.5.3.5/vasp'
else:
job_bin = job_bin
d = {
'type': 'SLURM',
'params': {
'nodes': str(nnodes),
'ntasks': str(nprocs),
'walltime': walltime,
'queue': 'normal',
'account': 'TG-DMR050028N',
'job_name': 'vasp_job',
'rocket_launch': 'ibrun ' + job_bin
}
}
# running henniggroup machines
elif hostname in ['hydrogen', 'helium', 'lithium', 'beryllium', 'carbon']:
job_cmd = [
'nohup', '/opt/openmpi_intel/bin/mpirun', '-n',
str(nprocs), job_bin
]
# test
else:
job_cmd = ['ls', '-lt']
if d:
return (CommonAdapter(d['type'], **d['params']), job_cmd)
else:
return (None, job_cmd)
def from_dict(cls, d):
incar = Incar.from_dict(d["incar"])
poscar = Poscar.from_dict(d["poscar"])
potcar = Potcar.from_dict(d["potcar"])
kpoints = Kpoints.from_dict(d["kpoints"])
qadapter = None
if d["qadapter"] is not None:
qadapter = CommonAdapter.from_dict(d["qadapter"])
return MPINTVaspInputSet(d["name"], incar, poscar, potcar,
kpoints, qadapter, **d["kwargs"])
def from_dict(cls, d):
incar = Incar.from_dict(d["incar"])
poscar = Poscar.from_dict(d["poscar"])
potcar = Potcar.from_dict(d["potcar"])
kpoints = Kpoints.from_dict(d["kpoints"])
qadapter = None
if d["qadapter"] is not None:
qadapter = CommonAdapter.from_dict(d["qadapter"])
script_name = d["script_name"]
return MPINTVaspInputSet(d["name"], incar, poscar, potcar,
kpoints, qadapter,
script_name=script_name,
vis_logger=logging.getLogger(d["logger"]),
**d["kwargs"])
def get_run_cmmnd(nnodes=1,
ntasks=16,
walltime='10:00:00',
job_bin=None,
job_name=None,
mem=None):
"""
returns the fireworks CommonAdapter based on the queue
system specified by mpint_config.yaml and the submit
file template also specified in mpint_config.yaml
NOTE: for the job_bin, please specify the mpi command as well:
Eg: mpiexec /path/to/binary
"""
d = {}
job_cmd = None
try:
qtemp_file = open(QUEUE_TEMPLATE + 'qtemplate.yaml')
qtemp = yaml.load(qtemp_file)
qtemp_file.close()
except:
# test case scenario
qtemp = {'account': None, 'mem': None, \
'walltime': '10:00:00', 'nodes': 1, 'pre_rocket': None, 'job_name': None, \
'ntasks': 16, 'email': None, 'rocket_launch': None}
qtemp.update({'nodes': nnodes, 'ntasks':ntasks, 'walltime': walltime, \
'rocket_launch': job_bin, 'job_name':job_name,'mem':mem})
# SLURM queue
if QUEUE_SYSTEM == 'slurm':
if job_bin is None:
job_bin = VASP_STD_BIN
else:
job_bin = job_bin
d = {'type': 'SLURM', 'params': qtemp}
# PBS queue
elif QUEUE_SYSTEM == 'pbs':
if job_bin is None:
job_bin = VASP_STD_BIN
else:
job_bin = job_bin
d = {'type': 'PBS', 'params': qtemp}
else:
job_cmd = ['ls', '-lt']
if d:
#print (CommonAdapter(d['type'], **d['params']), job_cmd)
return (CommonAdapter(d['type'], **d['params']), job_cmd)
else:
return (None, job_cmd)
val_spec = copy.deepcopy(spec)
val_spec['TRAINING_SEQUENCES'] = VALIDATION_SEQUENCES
val_spec['seq_idx_to_eval'] = VALIDATION_SEQUENCES
val_spec['validation_eval'] = True
val_batch = Firework(RunRBPF_Batch(), spec = val_spec)
val_eval = Firework(RunEval(), spec = val_spec)
storeResultsFW2 = Firework(StoreResultsInDatabase(), spec=val_spec)
first_iter= Firework(Iterate(), spec = spec)
workflow = Workflow([init_batch, eval_init, first_iter, val_batch, val_eval, storeResultsFW1, storeResultsFW2],
{init_batch: [eval_init], eval_init: [first_iter], val_batch: [val_eval], eval_init:[storeResultsFW1], val_eval:[storeResultsFW2]})
launchpad.add_wf(workflow)
qadapter = CommonAdapter.from_file("%sfireworks_files/my_qadapter.yaml" % RBPF_HOME_DIRECTORY)
# rapidfire(launchpad, FWorker(), qadapter, launch_dir='.', nlaunches='infinite', njobs_queue=20,
# njobs_block=500, sleep_time=None, reserve=False, strm_lvl='INFO', timeout=None,
# fill_mode=False)
fworker = FWorker()
rapidfire(launchpad, fworker, qadapter, launch_dir='.', nlaunches='infinite', njobs_queue=20,
njobs_block=500, sleep_time=None, reserve=False, strm_lvl="DEBUG", timeout=None,
fill_mode=False)
rlaunch_command = "rlaunch singleshot --offline"
else:
lpad_abspath = str(pathlib.Path("autogenerated_launchpad.yaml").resolve())
rlaunch_command = "rlaunch -l " + lpad_abspath + " singleshot"
print(rlaunch_command)
if IS_QUEUE:
dft = CommonAdapter(
q_type="SLURM",
queue="test",
nodes= 1,
ntasks= 8,
walltime= '00:05:00',
mem_per_cpu = 3000,
constraint='hsw',
account= None,
job_name= 'dfttestrun',
pre_rocket= "module load cp2k-env/4.1-hsw",
post_rocket= "echo 'current fashion: post-modern rocket after running dft'",
logdir= abspath,
rocket_launch= rlaunch_command
)
lightweight = CommonAdapter(
q_type="SLURM",
queue="test",
nodes= 1,
ntasks= 8,
walltime= '00:03:30',
constraint='hsw',
account= None,
default_queue_adapter_args = {
'q_type': 'PBS', # Queue adapter template to use.
'q_name': 'titan_queue', # FW name for the queue.
'template_file': os.path.join(src_dir, 'templates', 'pbs.template'),
'queue': 'batch', # PBS queue to use.
'account': PROJID, # OLCF allocation account.
'logdir': log_dir,
'nnodes': 0, # Overridden by tasks.
'ppnode': 0, # Overridden by tasks.
'walltime': '01:00:00', # Overridden by tasks.
'pre_rocket': 'module load python python_fireworks python_setuptools',
'rocket_launch': 'rlaunch singleshot --offline',
'job_name': 'unnamed_job',
}
default_queue = CommonAdapter(**default_queue_adapter_args)
#===============================================================================
# Generate fake jobs and a sample workflow.
from fireworks import Firework, ScriptTask
class Jobs:
def __init__(self, category, adapter):
self.fireworks = {}
self.adapter = dict(adapter)
self.category = category
self.spec = {
'_queueadapter': self.adapter,
'_category': self.category,
def get_run_cmmnd(nnodes=1,
ntasks=16,
walltime='24:00:00',
job_bin=None,
job_name=None):
"""
depends on the supercomputing faciltiy being used.
set a sample submit script in the fireworks directory which is
installed in your virtual environment as for example:
/my_venv/lib/python2.7/site-packages/FireWorks-1.2.5-py2.7.egg/
fireworks/user_objects/queue_adapters/
the keys to the dictionary d are the defaults on ufhpc's
hipergator2 supercomputing facility
"""
d = {}
job_cmd = None
queue = QUEUE_SYSTEM
hostname = socket.gethostname()
# FIXME: Using hostnames to determine behavior is terrible practice, as is
# hard-coding file directories.
# old hipergator which can be generalized into a pbs qdapter for fireworks
# if 'ufhpc_pbs' in hostname:
# if job_bin is None:
# job_bin = '/home/km468/Software/VASP/vasp.5.3.5/vasp'
# else:
# job_bin = job_bin
# d = {'type': 'PBS',
# 'params':
# {
# 'nnodes': str(nnodes),
# 'ppnode': str(int(nprocs / nnodes)),
# 'walltime': walltime,
# 'job_name': 'vasp_job',
# 'email': '*****@*****.**',
# 'notification_options': 'ae',
# 'pre_rocket': '#PBS -l pmem=' + str(mem) + 'mb',
# 'rocket_launch': 'mpirun ' + job_bin
# hipergator: currently hipergator2
if 'slurm' in queue:
#if 'ufhpc' in hostname:
if job_bin is None:
job_bin = STD_BINARY
else:
job_bin = job_bin
# FIXME: think of way to generalize this to a SLURM queue, some specs
# here are ufhpc dependent and depend on the submission script settings
# for the binary
d = {
'type': 'SLURM',
'params': {
'nodes': str(nnodes),
'ntasks': str(int(ntasks)),
'walltime': walltime,
'job_name': job_name,
'email': '*****@*****.**',
'notification_options': 'ae',
'pre_rocket': 'module load intel/2016.0.109 openmpi',
'rocket_launch': 'mpiexec ' + job_bin
}
}
# stampede
elif 'stampede' in hostname:
if job_bin is None:
job_bin = '/home1/01682/km468/Software/VASP/vasp.5.3.5/vasp'
else:
job_bin = job_bin
d = {
'type': 'SLURM',
'params': {
'nodes': str(nnodes),
'ntasks': str(ntasks),
'walltime': walltime,
'queue': 'normal',
'account': 'TG-DMR050028N',
'job_name': 'vasp_job',
'rocket_launch': 'ibrun ' + job_bin
}
}
# running henniggroup machines
elif hostname in ['hydrogen', 'helium', 'lithium', 'beryllium', 'carbon']:
job_cmd = [
'nohup', '/opt/openmpi_intel/bin/mpirun', '-n',
str(ntasks), job_bin
]
# test
else:
job_cmd = ['ls', '-lt']
if d:
return (CommonAdapter(d['type'], **d['params']), job_cmd)
else:
return (None, job_cmd)