def __init__(self,vasp_task=None,name='vaspfw',handlers=None, handler_params=None, config_file=None):
self.name = name
self.handlers=handlers if handlers else []
self.handler_params=handler_params if handler_params else {}
if config_file:
config_dict = loadfn(config_file)
elif os.path.exists(os.path.join(os.environ['HOME'], 'vasp_interface_defaults.yaml')):
config_dict = loadfn(os.path.join(os.environ['HOME'], 'vasp_interface_defaults.yaml'))
else:
config_dict = {}
if config_dict:
self.custodian_opts = config_dict.get('CUSTODIAN_PARAMS', {})
if self.custodian_opts.get('handlers', []):
self.handlers.extend(self.custodian_opts.get('handlers', []))
self.handler_params.update(self.custodian_opts.get('handler_params', {}))
self.tasks=[vasp_task.input,RunCustodianTask(handlers=self.handlers,
handler_params=self.handler_params)] if isinstance(vasp_task,
VaspInputInterface) else [vasp_task]
self.Firework=Firework(self.tasks,name=self.name)
# Try to establish connection with Launchpad
try:
self.LaunchPad=LaunchPad.from_file(os.path.join(os.environ["HOME"], ".fireworks", "my_launchpad.yaml"))
except:
self.LaunchPad = None
def get_fw_by_id(self, dbname, fw_id):
"""
Given a Firework id, give back a Firework object
:param fw_id: Firework id (int)
:return: Firework object
"""
return Firework.from_dict(self.get_fw_dict_by_id(fw_id))
def run_task(self, fw_spec):
objective_with_inc = fw_spec["%s_eval_metrics_with_inc" %
EVAL_SCRIPT][OBJECTIVE_METRIC]
objective_with_dec = fw_spec["%s_eval_metrics_with_dec" %
EVAL_SCRIPT][OBJECTIVE_METRIC]
orig_objective = fw_spec["%s_eval_metrics" %
EVAL_SCRIPT][OBJECTIVE_METRIC]
(param_name, row_idx, col_idx) = get_param(fw_spec['param_idx'])
if orig_objective >= objective_with_inc and orig_objective >= objective_with_dec:
fw_spec[param_name] = fw_spec['orig_param_val'] #update parameter
fw_spec["%s_alpha" % param_name][row_idx][
col_idx] *= alpha_dec #update parameter's alpha
best_obj = fw_spec["%s_eval_metrics" %
EVAL_SCRIPT][OBJECTIVE_METRIC]
change_for_best_obj = 'const'
elif objective_with_dec >= objective_with_inc and objective_with_dec > orig_objective:
fw_spec[param_name] = fw_spec['dec_param_val'] #update parameter
fw_spec["%s_alpha" % param_name][row_idx][
col_idx] *= alpha_inc #update parameter's alpha
fw_spec["%s_eval_metrics" % EVAL_SCRIPT] = fw_spec[
"%s_eval_metrics_with_dec" %
EVAL_SCRIPT] #update baseline metrics for next iteration
best_obj = fw_spec["%s_eval_metrics_with_dec" %
EVAL_SCRIPT][OBJECTIVE_METRIC]
change_for_best_obj = 'dec'
elif objective_with_inc > objective_with_dec and objective_with_inc > orig_objective:
fw_spec[param_name] = fw_spec['inc_param_val'] #update parameter
fw_spec["%s_alpha" % param_name][row_idx][
col_idx] *= alpha_inc #update parameter's alpha
fw_spec["%s_eval_metrics" % EVAL_SCRIPT] = fw_spec[
"%s_eval_metrics_with_inc" %
EVAL_SCRIPT] #update baseline metrics for next iteration
best_obj = fw_spec["%s_eval_metrics_with_inc" %
EVAL_SCRIPT][OBJECTIVE_METRIC]
change_for_best_obj = 'inc'
else:
print "Coding Error ChooseNextIter()"
print(objective_with_inc, objective_with_dec, orig_objective)
sys.exit(1)
fw_spec['param_idx'] = inc_parameter(fw_spec['param_idx'])
next_iter_firework = Firework(Iterate(), fw_spec)
return FWAction(stored_data={
'best_obj': best_obj,
'change_for_best_obj': change_for_best_obj,
'parameter_changed_name': param_name,
'parameter_changed_val': fw_spec[param_name]
},
additions=next_iter_firework)
def partial_fw(self):
if not self._fw:
fields = list(self.db_fields) + list(self.db_launch_fields)
data = self._fwc.find_one({'fw_id': self.fw_id}, projection=fields)
launch_data = {} # move some data to separate launch dict
for key in self.db_launch_fields:
launch_data[key] = data[key]
del data[key]
self._lids = launch_data
self._fw = Firework.from_dict(data)
return self._fw
def test_dagflow_input(self):
""" missing input """
fw2 = Firework(PyTask(func='math.pow',
inputs=['first power', 'exponent'],
outputs=['second power']),
name='pow(pow(2, 3), 4)')
wfl = Workflow([self.fw1, fw2], {self.fw1: [fw2], fw2: []})
msg = (r"An input field must have exactly one source', 'step', "
r"'pow(pow(2, 3), 4)', 'entity', 'exponent', 'sources', []")
with self.assertRaises(AssertionError) as context:
DAGFlow.from_fireworks(wfl)
self.assertTrue(msg in str(context.exception))
def setUp(self):
try:
__import__('igraph', fromlist=['Graph'])
except (ImportError, ModuleNotFoundError):
raise unittest.SkipTest(
'Skipping because python-igraph not installed')
self.fw1 = Firework(PyTask(func='math.pow',
inputs=['base', 'exponent'],
outputs=['first power']),
name='pow(2, 3)',
spec={
'base': 2,
'exponent': 3
})
self.fw2 = Firework(PyTask(func='math.pow',
inputs=['first power', 'exponent'],
outputs=['second power']),
name='pow(pow(2, 3), 4)',
spec={'exponent': 4})
self.fw3 = Firework(PyTask(func='print', inputs=['second power']),
name='the third one')
def test_directory(self):
doc_class = self.get_document_class_from_mixin(DirectoryMixin)
doc = doc_class()
# create and run a fireworks workflow
task = PyTask(func="time.sleep", args=[0.5])
wf = Workflow([
Firework(task, spec={'wf_task_index': "1"}, fw_id=1),
Firework(task, spec={'wf_task_index': "2"}, fw_id=2)
])
self.lp.add_wf(wf)
rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR)
wf = self.lp.get_wf_by_fw_id(1)
doc.set_dir_names_from_fws_wf(wf)
assert len(doc.dir_names) == 2
if has_mongodb():
doc.save()
query_result = doc_class.objects()
assert len(query_result) == 1
assert len(query_result[0].dir_names) == 2
def wf_creator(x):
"""
A workflow creator function returning a workflow of the following form:
simulation (fw1)
|
optimization (fw2)
Args:
x ([list]): A 3 vector of the form [int, float, str], where the elements
are constrained to the search space given in x_dim above.
Returns:
(Workflow): The workflow which will run the simulation and optimization
fireworks.
"""
simulation = Firework([ComplexMultiObjTask()],
spec={'_x': x},
name="simulation")
optimization = Firework([OptTask(**db_info)], name="optimization")
return Workflow([simulation, optimization], {simulation: optimization})
def main(stocks):
assert stocks
start_date = datetime.date.today() - datetime.timedelta(LastDaysRaw)
# connect to google sheets
dailydb = GspreadDB(DailyDbName, 'DAILY', JsonAuthFile)
stockdb = GspreadDB(SourceDbName, 'IOPV', JsonAuthFile)
iopv_by_date = {}
daily_by_date = {}
for stock in stocks:
print("Processing %s" % stock)
try:
ticker = stockdb.getstockticker(stock)
tickersheet = stockdb.getstocksheet(stock)
rows = tickersheet.get_all_records()
for row in rows:
time = to_datetime(row['TIME'], format='%d/%m/%Y %H:%M:%S')
if time.date() < start_date:
continue
date = str(time)
if date not in iopv_by_date:
iopv_by_date[date] = {'DATE': date, 'IOPV': {}}
iopv_by_date[date]['IOPV'][ticker] = row['IOPV']
dailysheet = dailydb.getstocksheet(stock)
rows = dailysheet.get_all_records()
for row in rows:
date = to_datetime(row['DATE'], format='%d/%m/%Y')
date = str(date.date())
if date not in daily_by_date:
daily_by_date[date] = {'DATE': date, 'IOPV': {}}
ohlc = dict([(x, row[x])
for x in ('OPEN', 'HIGH', 'LOW', 'CLOSE')])
daily_by_date[date]['IOPV'][ticker] = ohlc
except ValueError as ve:
print(f"error getting sheet data for '{stock}': {ve}")
continue
print(f'Writing JSON files...')
with open('iopv-raw.json', 'w') as fp:
json.dump(iopv_by_date, fp, sort_keys=True, indent=4)
with open('iopv-daily.json', 'w') as fp:
json.dump(daily_by_date, fp, sort_keys=True, indent=4)
if update_fire:
fire = Firework()
# update raw database
print(f'Updating Firebase raw database...')
for date, rec in iopv_by_date.items():
fire.update_stock_raw(date, rec)
# update daily database
print(f'Updating Firebase daily database...')
for date, rec in daily_by_date.items():
fire.update_stock_daily(date, rec)
def generate_nanoclusters(n_initial_configurations,
n_configurations,
shape,
nanocluster_size,
compositions,
elements,
generate_pure_nanoclusters=True,
bondlength_dct={}):
"""
Firework to generate binary nanoclusters of defined size and shape.
For each binary element combination, for each composition, n_configurations
maximally dissimilar structures are created and uploaded to the
simulations collection of the mongodb database.
For further information on the generation algorithm,
consult the documentation of cluskit.
The new structures are added to the simulation collection of
the mongodb database.
Args:
n_initial_configurations (int) : number of initial configurations per
composition to choose from (higher number
will make the grid finer)
n_configurations (int) : number of configurations per composition
(chosen based on maximally different
structural features)
shape (str) : determines shape of nanoclusters. 'ico', 'octa' and 'wulff'
nanocluster_size (int) : determines nanocluster size. Meaning depends on shape
compositions (list) : each element determines the amount of atoms of type 1.
elements (list) : elements (str) to iterate over
generate_pure_nanoclusters (bool) : if set to True, also pure
nanoclusters are generated
bondlength_dct (dict) : bond lengths to use for specific elements.
Some default bond lenghts are provided for
common elements
Returns:
Firework : Firework NCGenerationWork
"""
firetask1 = NCGenerationTask(
n_initial_configurations=n_initial_configurations,
n_configurations=n_configurations,
shape=shape,
nanocluster_size=nanocluster_size,
compositions=compositions,
elements=elements,
generate_pure_nanoclusters=True,
bondlength_dct={})
dct = {'_category': "large", 'name': 'NCGenerationTask'}
fw = Firework([firetask1], spec=dct, name='NCGenerationWork')
return fw
def setUp(self):
super(TestElasticWorkflow, self).setUp()
self.tf_loc = os.path.join(ref_dir, 'elastic_wf')
self.struct_si = PymatgenTest.get_structure("Si")
self.struct_si = SpacegroupAnalyzer(
self.struct_si).get_conventional_standard_structure()
self.opt_struct = Structure.from_file(
os.path.join(self.tf_loc, '1', 'inputs', 'POSCAR'))
# Base WF
self.base_wf = get_wf(self.struct_si,
"optimize_only.yaml",
params=[{
"db_file": ">>db_file<<"
}])
self.base_wf.append_wf(
get_wf_elastic_constant(self.struct_si,
db_file='>>db_file<<',
stencils=[[0.01]] * 3 + [[0.03]] * 3,
copy_vasp_outputs=True),
self.base_wf.leaf_fw_ids)
self.base_wf_noopt = get_wf_elastic_constant(self.opt_struct,
stencils=[[0.01]] * 3 +
[[0.03]] * 3,
copy_vasp_outputs=False,
sym_reduce=False,
db_file='>>db_file<<')
ec_incar_update = {'incar_update': {'EDIFF': 1e-6, 'ENCUT': 700}}
self.base_wf = add_modify_incar(self.base_wf, ec_incar_update)
self.base_wf_noopt = add_modify_incar(self.base_wf_noopt,
ec_incar_update)
# Full preset WF
self.preset_wf = wf_elastic_constant(self.struct_si)
# Minimal WF
self.minimal_wf = wf_elastic_constant_minimal(self.opt_struct)
self.minimal_wf = add_modify_incar(self.minimal_wf, ec_incar_update)
# TOEC WF (minimal)
self.toec_wf = wf_elastic_constant_minimal(self.struct_si, order=3)
self.toec_wf = add_modify_incar(self.toec_wf, ec_incar_update)
toec_data = loadfn(os.path.join(self.tf_loc, 'toec_data.json'))
# Rather than run entire workflow, preload the spec to test the analysis
toec_analysis = Firework(
[
ElasticTensorToDb(
structure=self.struct_si, order=3, db_file=">>db_file<<")
],
spec={"deformation_tasks": toec_data['deformation_tasks']})
self.toec_analysis = Workflow([toec_analysis])
def submitVib(self):
from fireworks import Firework, Workflow
from standardScripts import Vibrations, SaveResults
clusters = self.allocate(2)
timestamp = '_' + datetime.now().strftime('%Y_%m_%d_%H_%M')
names = [x + '_%d' % self.jobid for x in ['Vibrations', 'SaveResults']]
fw1 = Firework(
[OptimizeLattice()],
name=names[0],
spec={
"jobID": self.jobid,
'_fworker': clusters[0].fworker,
"_pass_job_info": True,
"_files_out": {
"fw1": "vibrations.txt"
},
"_queueadapter":
clusters[0].qfunc(self.guessTime('Vibrations')),
"_launch_dir": clusters[0].launchdir + names[0] + timestamp
})
fw2 = Firework(
[SaveResults()],
name=names[1],
parents=[fw1, fw2, fw3],
spec={
"jobID": self.jobid,
'_fworker': clusters[1].fworker #MUST be sherlock
,
"_files_in": {
"fw1": "vibrations.txt"
},
"_queueadapter":
clusters[1].qfunc(self.guessTime('SaveResults')),
"_launch_dir": clusters[1].launchdir + names[1] + timestamp
})
return Workflow([fw1, fw2], name='Vibration_%d' % self.jobid)
def add_task(self, task):
'''
Function used to add another FireTask to the Firework. If
given task is a VaspInputInterface Object, it will create the WriteInputTask
as well as add another RunCustodianTask() to the FireTask list.
'''
if isinstance(task, VaspInputInterface):
self.tasks.extend([task.input,
RunCustodianTask(handlers=self.handlers,
handler_params=self.handler_params)])
else:
self.tasks.append(task)
self.Firework=Firework(self.tasks,name=self.name)
def test_checkout_fw(self):
os.chdir(MODULE_DIR)
self.lp.add_wf(
Firework(ScriptTask.from_str(
shell_cmd='echo "hello 1"',
parameters={"stdout_file": "task.out"}),
fw_id=1))
self.lp.add_wf(
Firework(ScriptTask.from_str(
shell_cmd='echo "hello 2"',
parameters={"stdout_file": "task.out"}),
fw_id=2))
launch_multiprocess(self.lp, FWorker(), 'DEBUG', 0, 2, 10)
fw1 = self.lp.get_fw_by_id(1)
fw2 = self.lp.get_fw_by_id(2)
self.assertEqual(fw1.launches[0].state_history[-1]["state"],
"COMPLETED")
self.assertEqual(fw2.launches[0].state_history[-1]["state"],
"COMPLETED")
with open(os.path.join(fw1.launches[0].launch_dir, "task.out")) as f:
self.assertEqual(f.readlines(), ['hello 1\n'])
with open(os.path.join(fw2.launches[0].launch_dir, "task.out")) as f:
self.assertEqual(f.readlines(), ['hello 2\n'])
def setUp(self):
fireworks.core.firework.EXCEPT_DETAILS_ON_RERUN = True
self.error_test_dict = {'error': 'description', 'error_code': 1}
fw = Firework(ScriptTask.from_str('echo "test offline"',
{'store_stdout': True}),
name="offline_fw",
fw_id=1)
self.lp.add_wf(fw)
self.launch_dir = os.path.join(MODULE_DIR, "launcher_offline")
os.makedirs(self.launch_dir)
self.old_wd = os.getcwd()
def parametersNotValid(self, model):
"""
"""
try:
# Continue with exact tasks, parameter estimation and (finally) this
# task in order to resume normal operation
wf = model.initialisationStrategy().workflow(model)
wf.addAfterAll(Workflow2([Firework(self)], name='original task'))
return FWAction(detours=wf)
except:
import traceback
traceback.print_exc()
return FWAction(defuse_children=True)
def chunk_calculations(template,
target_path,
chunk_size=-1,
name="cp2k_run_id",
n_max_restarts=4,
simulation_method="cp2k",
skip_dft=False,
is_safeguard=True):
"""
Create Fireworks with new calculations to setup and run.
Args:
template (str) : input file for calculations represented as string.
It works as a template which is later modified by the
simulation-specific Firework.
target_path (str) : absolute path to the target directory
(needs to exist) on the computing resource.
name (str) : individual calculation folder name
is prefixed with the given string
n_max_restarts (int) : number of times the calculation is restarted upon failure
chunk_size (int) : det : number of calculations to be run simulataneously. Default -1
means all calculations are run at once.
simulation_method (str) : Specifies which simulation code to use.
Currently, only CP2K is implemented.
skip_dft (bool) : If set to true, the simulation step is skipped in all
following simulation runs. Instead the structure is returned unchanged.
is_safeguard (bool) : if False, the workflow is not paused when not all CP2K jobs
converge properly after the maximum number of restarts.
Returns:
Firework : StructureFolderWork Firework,
creates new Fireworks as detours from workflow
"""
firetask1 = ChunkCalculationsTask(
template=template,
target_path=target_path,
chunk_size=chunk_size,
name=name,
n_max_restarts=n_max_restarts,
simulation_method=simulation_method,
skip_dft=skip_dft,
is_safeguard=is_safeguard,
)
fw = Firework([firetask1],
spec={
'_category': "medium",
'name': 'ChunkCalculationsTask'
},
name='ChunkCalculationsWork')
return fw
def get_wf_bulk_modulus(structure,
vasp_input_set=None,
vasp_cmd="vasp",
deformations=None,
db_file=None,
user_kpoints_settings=None,
eos="vinet"):
"""
Returns the workflow that computes the bulk modulus by fitting to the given equation of state.
Args:
structure (Structure): input structure.
vasp_input_set (VaspInputSet)
vasp_cmd (str): vasp command to run.
deformations (list): list of deformation matrices(list of lists).
db_file (str): path to the db file.
user_kpoints_settings (dict): example: {"grid_density": 7000}
eos (str): equation of state used for fitting the energies and the volumes.
supported equation of states: "quadratic", "murnaghan", "birch", "birch_murnaghan",
"pourier_tarantola", "vinet", "deltafactor". See pymatgen.analysis.eos.py
Returns:
Workflow
"""
tag = datetime.utcnow().strftime('%Y-%m-%d-%H-%M-%S-%f')
deformations = [Deformation(defo_mat) for defo_mat in deformations]
wf_bulk_modulus = get_wf_deformations(
structure,
deformations,
name="bulk_modulus deformation",
vasp_input_set=vasp_input_set,
lepsilon=False,
vasp_cmd=vasp_cmd,
db_file=db_file,
user_kpoints_settings=user_kpoints_settings,
tag=tag)
fw_analysis = Firework(FitEquationOfStateTask(tag=tag,
db_file=db_file,
eos=eos),
name="fit equation of state")
append_fw_wf(wf_bulk_modulus, fw_analysis)
wf_bulk_modulus.name = "{}:{}".format(
structure.composition.reduced_formula, "Bulk modulus")
return wf_bulk_modulus
def initiate_cluster(inputs):
# check how many image folders are there
contents_list = multi_call(inputs)
lpad = LaunchPad(**yaml.load(open(join(celltkroot, "fireworks", "my_launchpad.yaml"))))
wf_fws = []
for contents in contents_list:
fw_name = "cluster_celltk"
fw = Firework(clustercelltk(contents=contents),
name = fw_name,
spec = {"_queueadapter": {"job_name": fw_name, "walltime": "47:00:00"}},
)
wf_fws.append(fw)
# end loop over input values
workflow = Workflow(wf_fws, links_dict={})
lpad.add_wf(workflow)
def executeAndCatchExceptions(self, op, text):
"""
@brief Method executing tasks and catching callbacks
@details
The
"""
try:
op()
except OutOfBounds as e:
model = e.args[1]
print(
term.cyan +
'%s out-of-bounds, executing outOfBoundsStrategy for model %s'
% (text, model._id) + term.normal)
# Continue with exact tasks, parameter estimation and (finally) this
# task in order to resume normal operation
wf = model.outOfBoundsStrategy().workflow(
model, outsidePoint=model.outsidePoint)
wf.append_wf(Workflow([Firework(self)], name='original task'),
wf.leaf_fw_ids)
raise ModifyWorkflow(FWAction(detours=wf))
except ParametersNotValid as e:
model = e.args[1]
print(term.cyan + '%s is not initialised, ' % text +
'executing initialisationStrategy for model %s' % model._id +
term.normal)
# Continue with exact tasks, parameter estimation and (finally) this
# task in order to resume normal operation
wf = model.initialisationStrategy().workflow(model)
wf.append_wf(Workflow([Firework(self)], name='original task'),
wf.leaf_fw_ids)
raise ModifyWorkflow(FWAction(detours=wf))
def test_run(self):
create_fw = Firework(
[mock_objects.CreateOutputsTask(extensions=["WFK", "DEN"])],
fw_id=1)
delete_fw = Firework([FinalCleanUpTask(["WFK", "1WF"])],
parents=create_fw,
fw_id=2,
spec={"_add_launchpad_and_fw_id": True})
wf = Workflow([create_fw, delete_fw])
self.lp.add_wf(wf)
rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR, nlaunches=1)
# check that the files have been created
create_fw = self.lp.get_fw_by_id(1)
create_ldir = create_fw.launches[0].launch_dir
for d in ["tmpdata", "outdata", "indata"]:
assert os.path.isfile(os.path.join(create_ldir, d, "tmp_WFK"))
assert os.path.isfile(os.path.join(create_ldir, d, "tmp_DEN"))
rapidfire(self.lp, self.fworker, m_dir=MODULE_DIR, nlaunches=1)
wf = self.lp.get_wf_by_fw_id(1)
assert wf.state == "COMPLETED"
for d in ["tmpdata", "indata"]:
assert not os.path.isfile(os.path.join(create_ldir, d, "tmp_WFK"))
assert not os.path.isfile(os.path.join(create_ldir, d, "tmp_DEN"))
assert not os.path.isfile(
os.path.join(create_ldir, "outdata", "tmp_WFK"))
assert os.path.isfile(os.path.join(create_ldir, "outdata", "tmp_DEN"))
def setUp(self):
fireworks.core.firework.EXCEPT_DETAILS_ON_RERUN = True
self.error_test_dict = {'error': 'description', 'error_code': 1}
fw = Firework([
ExecutionCounterTask(),
ScriptTask.from_str('date +"%s %N"',
parameters={'stdout_file': 'date_file'}),
ExceptionTestTask(exc_details=self.error_test_dict)
])
self.lp.add_wf(fw)
ExecutionCounterTask.exec_counter = 0
ExceptionTestTask.exec_counter = 0
self.old_wd = os.getcwd()
def make_firework(atoms, fw_name, vasp_settings):
'''
This function makes a FireWorks rocket to perform a VASP relaxation
Args:
atoms `ase.Atoms` object to relax
fw_name Dictionary of tags/etc to use as the FireWorks name
vasp_settings Dictionary of VASP settings to pass to Vasp()
Returns:
firework An instance of a `fireworks.Firework` object that is set up
to perform a VASP relaxation
'''
# Warn the user if they're submitting a big one
if len(atoms) > 80:
warnings.warn(
'You are making a firework with %i atoms in it. This may '
'take awhile.' % len(atoms), RuntimeWarning)
# Take the `vasp_functions` submodule in GASpy and then pass it out to each
# FireWork rocket to use.
vasp_filename = vasp_functions.__file__
if vasp_filename.split(
'.')[-1] == 'pyc': # Make sure we use the source file
vasp_filename = vasp_filename[:-3] + 'py'
with open(vasp_filename) as file_handle:
vasp_functions_contents = file_handle.read()
pass_vasp_functions = FileWriteTask(
files_to_write=[{
'filename': 'vasp_functions.py',
'contents': vasp_functions_contents
}])
# Convert the atoms object to a string so that we can pass it through
# FireWorks, and then tell the FireWork rocket to use our `vasp_functions`
# submodule to unpack the string
atom_trajhex = encode_atoms_to_trajhex(atoms)
read_atoms_file = PyTask(func='vasp_functions.hex_to_file',
args=['slab_in.traj', atom_trajhex])
# Tell the FireWork rocket to perform the relaxation
relax = PyTask(func='vasp_functions.runVasp',
args=['slab_in.traj', 'slab_relaxed.traj', vasp_settings],
stored_data_varname='opt_results')
fw_name['user'] = getpass.getuser()
firework = Firework([pass_vasp_functions, read_atoms_file, relax],
name=fw_name)
return firework
def wf_creator_accuracy(x, launchpad):
"""
An expensive test ensuring the default predictor actually performs better
than the average random case on the function defined in AccuracyTask.
"""
spec = {'_x_opt': x}
dims = [(1, 10), (10.0, 20.0), (20.0, 30.0)]
at = AccuracyTask()
ot = OptTask(wf_creator='rocketsled.tests.tests.wf_creator_accuracy',
dimensions=dims,
lpad=launchpad,
wf_creator_args=[launchpad],
opt_label='test_accuracy',
maximize=True)
firework1 = Firework([at, ot], spec=spec)
return Workflow([firework1])
def wf_creator_parallel(x, launchpad):
"""
An expensive test ensuring the database is locked and released
correctly during optimization.
"""
spec = {'_x_opt': x}
dims = [(1, 5), (1, 5), (1, 5)]
at = AccuracyTask()
ot = OptTask(wf_creator='rocketsled.tests.tests.wf_creator_parallel',
dimensions=dims,
lpad=launchpad,
wf_creator_args=[launchpad],
opt_label='test_parallel',
maximize=True)
firework1 = Firework([at, ot], spec=spec)
return Workflow([firework1])
def run_task(self, fw_spec):
(param_name, det_name, idx) = get_param(fw_spec['param_idx'], fw_spec)
fw_spec['orig_param_val'] = fw_spec[param_name]
fw_spec['coord_ascent_iter'] += 1
#run an RBPF batch with the original parameter (avoid randomly getting a
#great result at some point in the past)
orig_spec = copy.deepcopy(fw_spec)
orig_spec['mod_direction'] = 'orig'
orig_batch_firework = Firework(RunRBPF_Batch(), spec=orig_spec)
#evaluate the RBPF batch with the original parameter
eval_orig_spec = copy.deepcopy(orig_spec)
eval_orig_spec['seq_idx_to_eval'] = eval_orig_spec[
'TRAINING_SEQUENCES']
orig_eval = Firework(RunEval(), spec=eval_orig_spec)
#run an RBPF batch with the parameter increased
inc_spec = copy.deepcopy(fw_spec)
inc_param_value = modify_parameter(inc_spec, 'inc')
inc_spec['inc_param_val'] = inc_param_value
fw_spec['inc_param_val'] = inc_param_value
inc_batch_firework = Firework(RunRBPF_Batch(), spec=inc_spec)
#evaluate the RBPF batch with the parameter increased
eval_inc_spec = copy.deepcopy(inc_spec)
eval_inc_spec['seq_idx_to_eval'] = eval_inc_spec['TRAINING_SEQUENCES']
inc_eval = Firework(RunEval(), spec=eval_inc_spec)
#run an RBPF batch with the parameter decreased
dec_spec = copy.deepcopy(fw_spec)
dec_param_value = modify_parameter(dec_spec, 'dec')
dec_spec['dec_param_val'] = dec_param_value
fw_spec['dec_param_val'] = dec_param_value
dec_batch_firework = Firework(RunRBPF_Batch(), spec=dec_spec)
#evaluate the RBPF batch with the parameter decreased
eval_dec_spec = copy.deepcopy(dec_spec)
eval_dec_spec['seq_idx_to_eval'] = eval_dec_spec['TRAINING_SEQUENCES']
dec_eval = Firework(RunEval(), spec=eval_dec_spec)
#run a firework that compares the evaluation metric when the parameter is increased, decreased,
#and its original value, and runs the next iteration of coordinate descent
next_iter = Firework(ChooseNextIter(), spec=fw_spec)
#Chain the fireworks together with proper dependencies and set 'em off!
iteration_workflow = Workflow([inc_batch_firework, dec_batch_firework, orig_batch_firework, \
inc_eval, dec_eval, orig_eval, next_iter],
{inc_batch_firework: [inc_eval], dec_batch_firework: [dec_eval], \
orig_batch_firework: [orig_eval], inc_eval:[next_iter], dec_eval:[next_iter], \
orig_eval:[next_iter]})
return FWAction(additions=iteration_workflow)
def wflow(self):
from fireworks import Firework, Workflow
cluster = self.allocate(1)[0]
timestamp = '_' + d.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
fw1 = Firework(
[self.standardScript()],
name=self.jobkind,
spec={
'params': self.params,
'_fworker': cluster.fworker,
'_queueadapter': cluster.qfunc(self.guessTime()),
'_launch_dir': cluster.launchdir + self.jobkind + timestamp
})
return Workflow([fw1], name=self.jobkind)
def wf_custom_predictor(x, launchpad):
"""
Testing a custom predictor which returns the same x vector for every guess,
using same workflow as test_basic.
"""
spec = {'_x_opt': x}
dims = [(1, 10), (10.0, 20.0), ['blue', 'green', 'red', 'orange']]
bt = BasicTestTask()
ot = OptTask(wf_creator='rocketsled.tests.tests.wf_custom_predictor',
dimensions=dims,
predictor='rocketsled.tests.tests.custom_predictor',
lpad=launchpad,
wf_creator_args=[launchpad],
opt_label='test_custom_predictor')
firework1 = Firework([bt, ot], spec=spec)
return Workflow([firework1])
def wf_creator(x):
spec = {'_x_opt': x, '_add_launchpad_and_fw_id': True}
Z_dim = dims
firework1 = Firework([
SumTask(),
OptTask(wf_creator='rocketsled.examples.parallel.wf_creator',
dimensions=Z_dim,
host='localhost',
port=27017,
name='rsled',
duplicate_check=True,
opt_label="opt_parallel")
],
spec=spec)
return Workflow([firework1])
def wf_creator_basic(x, launchpad):
"""
Testing a basic workflow with one Firework, and two FireTasks.
"""
spec = {'_x_opt': x}
dims = [(1, 10), (10.0, 20.0), ['blue', 'green', 'red', 'orange']]
bt = BasicTestTask()
ot = OptTask(wf_creator='rocketsled.tests.tests.wf_creator_basic',
dimensions=dims,
predictor='RandomForestRegressor',
predictor_kwargs={'random_state': 1},
lpad=launchpad,
wf_creator_args=[launchpad],
opt_label='test_basic')
firework1 = Firework([bt, ot], spec=spec)
return Workflow([firework1])
def wflow(self):
from fireworks import Firework, Workflow
from standardScripts import GetXCcontribs
cluster = self.allocate(1)[0]
timestamp = '_' + d.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
name = 'GetXCcontribs'
fw1 = Firework(
[GetXCcontribs()],
name=name,
spec={
'params': self.params,
'_fworker': cluster.fworker,
'_add_launchpad_and_fw_id': True,
"_queueadapter": cluster.qfunc(self.guessTime()),
"_launch_dir": cluster.launchdir + name + timestamp
})
return Workflow([fw1], name=name)
def test_dagflow_missing_input(self):
"""missing input"""
from fireworks.utilities.dagflow import DAGFlow
fw2 = Firework(
PyTask(func="math.pow",
inputs=["first power", "exponent"],
outputs=["second power"]),
name="pow(pow(2, 3), 4)",
)
wfl = Workflow([self.fw1, fw2], {self.fw1: [fw2], fw2: []})
msg = (
r"Every input in inputs list must have exactly one source.', 'step', "
r"'pow(pow(2, 3), 4)', 'entity', 'exponent', 'sources', []")
with self.assertRaises(AssertionError) as context:
DAGFlow.from_fireworks(wfl).check()
self.assertTrue(msg in str(context.exception))
def wf_creator_duplicates(x, launchpad):
"""
Test workflow for duplicate checking with tolerances.
"""
spec = {'_x_opt': x}
dims = [(1, 10), (10.0, 20.0), ['blue', 'green', 'red', 'orange']]
bt = BasicTestTask()
ot = OptTask(wf_creator='rocketsled.tests.tests.wf_creator_duplicates',
dimensions=dims,
predictor='rocketsled.tests.tests.custom_predictor',
lpad=launchpad,
duplicate_check=True,
tolerances=[0, 1e-6, None],
wf_creator_args=[launchpad],
opt_label='test_duplicates')
firework1 = Firework([bt, ot], spec=spec)
return Workflow([firework1])
def wf_creator_multiobjective(x, launchpad):
"""
Testing a multiobjective optimization.
"""
spec = {'_x_opt': x}
dims = [(1, 10), (10.0, 20.0), ['blue', 'green', 'red', 'orange']]
mt = MultiTestTask()
ot = OptTask(wf_creator='rocketsled.tests.tests.wf_creator_multiobjective',
dimensions=dims,
predictor='RandomForestRegressor',
predictor_kwargs={'random_state': 1},
lpad=launchpad,
wf_creator_args=[launchpad],
opt_label='test_multi')
firework1 = Firework([mt, ot], spec=spec)
return Workflow([firework1])
def get_fw_by_id(self, fw_id):
"""
Given a Firework id, give back a Firework object
:param fw_id: Firework id (int)
:return: Firework object
"""
fw_dict = self.fireworks.find_one({'fw_id': fw_id})
if not fw_dict:
raise ValueError('No Firework exists with id: {}'.format(fw_id))
# recreate launches from the launch collection
fw_dict['launches'] = list(self.launches.find(
{'launch_id': {"$in": fw_dict['launches']}}))
fw_dict['archived_launches'] = list(self.launches.find(
{'launch_id': {"$in": fw_dict['archived_launches']}}))
return Firework.from_dict(fw_dict)
from fireworks import Firework, LaunchPad, ScriptTask
from fireworks.core.rocket_launcher import launch_rocket, rapidfire
# set up the LaunchPad and reset it
launchpad = LaunchPad(strm_lvl='WARNING') # set messaging lowest level to WARNING
launchpad.reset('', require_password=False)
# create the Firework consisting of a single task
firetask = ScriptTask.from_str('cd /projects/development/LDRDSANS/fireworks/localtest; ./test_cluster.sh')
firework = Firework(firetask)
fw_yaml = firework.to_file("my_firework.yaml") # save to yaml file, and get the string representation
fw_json = firework.to_file("my_firework.json") # save to json file, and get the string representation
# store workflow and launch it locally
launchpad.add_wf(firework)
launch_rocket(launchpad) # same as "rlaunch singleshot"
#rapidfire(launchpad, FWorker(), strm_lvl='WARNING') # same as "rlaunch rapidfire"
# loading from file
# any class in FireWorks that subclasses FWSerializable has methods from_file and to_file
#firework = Firework.from_file("fw_test.yaml")
#fw_yaml = Firework.from_file("fw_test.json")
class VaspFirework():
"""
This is an interface to Fireworks for VaspInputInterface workflows
Required Params:
-vasp_task (obj): FireTask Object used to create a Firework.
2 FireTasks are automatically created upon constructing
the VaspFirework object from a WriteVaspInputTask:
[<WriteVaspInputTask()>, <RunCustodianTask()>].
Optional Params:
-name (str): Name given to the Firework
"""
def __init__(self,vasp_task=None,name='vaspfw',handlers=None, handler_params=None, config_file=None):
self.name = name
self.handlers=handlers if handlers else []
self.handler_params=handler_params if handler_params else {}
if config_file:
config_dict = loadfn(config_file)
elif os.path.exists(os.path.join(os.environ['HOME'], 'vasp_interface_defaults.yaml')):
config_dict = loadfn(os.path.join(os.environ['HOME'], 'vasp_interface_defaults.yaml'))
else:
config_dict = {}
if config_dict:
self.custodian_opts = config_dict.get('CUSTODIAN_PARAMS', {})
if self.custodian_opts.get('handlers', []):
self.handlers.extend(self.custodian_opts.get('handlers', []))
self.handler_params.update(self.custodian_opts.get('handler_params', {}))
self.tasks=[vasp_task.input,RunCustodianTask(handlers=self.handlers,
handler_params=self.handler_params)] if isinstance(vasp_task,
VaspInputInterface) else [vasp_task]
self.Firework=Firework(self.tasks,name=self.name)
# Try to establish connection with Launchpad
try:
self.LaunchPad=LaunchPad.from_file(os.path.join(os.environ["HOME"], ".fireworks", "my_launchpad.yaml"))
except:
self.LaunchPad = None
def add_task(self, task):
'''
Function used to add another FireTask to the Firework. If
given task is a VaspInputInterface Object, it will create the WriteInputTask
as well as add another RunCustodianTask() to the FireTask list.
'''
if isinstance(task, VaspInputInterface):
self.tasks.extend([task.input,
RunCustodianTask(handlers=self.handlers,
handler_params=self.handler_params)])
else:
self.tasks.append(task)
self.Firework=Firework(self.tasks,name=self.name)
def to_file(self,file_name):
self.Firework.to_file(file_name)
def add_fw_to_launchpad(self):
if self.LaunchPad:
self.Workflow=Workflow([self.Firework])
self.LaunchPad.add_wf(self.Workflow)
else:
print("No connection to LaunchPad. \n"
"Use 'to_file(<filename>)' to write a yaml file\n"
"to manually add Firework to LaunchPad later.\n")
def copy_files_from_previous(self, *args, **kwargs):
'''
Function used to automatically insert VASPTransferTask
between WriteVaspInputTask() and RunCustodianTask() in the
FireTask list. This is used to copy files from a previous
VASP Firework Calculation or from a known directory.
Known directories require the full path of the directory.
This function should not be used to copy files from a
previous Firetask within the same Firework.
Parameters:
-mode (str): Mode to use in transferring files:
move, mv, copy, cp, copy2, copytree, copyfile
Default: 'copy'
-ignore_errors (bool): Whether to quit upon error.
Default: True
-dir (str): Directory to transfer files from if not copying
directly from a previous Firework
Default: None
Example:
fw2.copy_files_from_previous('file1', 'file2', ... , 'filen', mode='copy', ignore_errors=False)
Example to copy files from directory other than previous Firework:
fw2.copy_files_from_previous('file1', 'file2', ... , 'filen',
dir='/path/to/dir', mode='copy', ignore_errors=False)
'''
mode = kwargs.get('mode', 'copy')
ignore_errors = kwargs.get('ignore_errors', True)
files = args[0] if isinstance(args[0], list) else [k for k in args]
self.dir = kwargs.get('dir', None)
if self.dir:
self.transfer=VaspTransferTask(files=files, dir=self.dir,
mode=mode, ignore_errors=ignore_errors)
else:
self.transfer=VaspTransferTask(files=files, mode=mode, ignore_errors=ignore_errors)
if isinstance(self.tasks[-1], RunCustodianTask):
self.tasks.pop(-1)
self.add_task(self.transfer)
self.add_task(RunCustodianTask(handlers=self.handlers,
handler_params=self.handler_params))
else:
self.add_task(self.transfer)
def add_handler(self, handler, **kwargs):
'''
Member function to add handler and handler options to
all Fireworks in the VaspFirework.tasks list.
Example (assuming fw1 is the firework object):
fw1.add_handler('WalltimeHandler', wall_time=3600, buffer_time=300)
fw1.add_handler('FrozenJobErrorHandler')
'''
for i,j in enumerate(self.Firework.tasks):
if isinstance(j, RunCustodianTask):
cust_handlers = j.get('handlers', [])
cust_params = j.get('handler_params', {})
if handler not in cust_handlers:
j['handlers'].append(handler)
j['handler_params'].update(kwargs)
self.Firework.spec['_tasks']=[t.to_dict() for t in
self.Firework.tasks] #re-initialize FW.spec
