def qadapter(qadapter_file=None, fworker_name="base"): """ Script to set up the configuration of the queueing system. Note that we store the queue_adapter in the same configuration directory as the Fireworker, i.e. fworker. This is in the assumption that each worker has one preferred queueing system. Args: qadapter_file (str): my_qadapter_file.yaml from which to configure the queue adapter. fworker_name (str): Which fworker configuration to add the queue adapter to. Note that there has to be a corresponding job_template.sh file for the queue adapter to adjust before submitting it to the fireworker queue. Returns: None """ if qadapter_file: queue_adapter = CommonAdapter.from_file(qadapter_file) else: logdir = input( "Please provide the path to the directory where the log " "should be stored. \n" "(Note: if the directory does not exist, it will be created): ") logdir = os.path.abspath(logdir) if not os.path.exists(logdir): os.makedirs(logdir) queue_adapter = CommonAdapter.from_file( os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples", "config", "fworker", "example_qadapter.yaml")) queue_adapter["logdir"] = logdir print( "\nNote: 'rocket_launch' has been set to an infinite rapidfire mode." ) try: os.makedirs( os.path.join(os.path.expanduser("~"), ".workflow_config", "fworker")) except FileExistsError: pass config_q_file = os.path.join(os.path.expanduser("~"), ".workflow_config", "fworker", fworker_name + "_qadapter.yaml") template_file = os.path.join(os.path.expanduser("~"), ".workflow_config", "fworker", fworker_name + "_job_template.sh") if not os.path.exists(template_file): print() warn_message = "No corresponding template file found! Don't forget to use " \ "'vsc config jobscript' to add the template file of the '" \ + fworker_name + "' fireworker." warnings.warn(warn_message) queue_adapter.template_file = template_file queue_adapter.to_file(config_q_file) print("\nQueue adapter file written to " + config_q_file)
def load_config(config, name="base"): """ Load a LaunchPad, FWorker of QueueAdapter from the configuration files. Args: config (str): Type of configuration file to load. Either "launchpad", "fworker" or "qadapter". name (str): Name of the configuration. Defaults to "base". Returns: Either a LaunchPad, FWorker or QueuAdapter, depending on the "config" argument. """ try: if config == "launchpad": return LaunchPad.from_file( os.path.join(os.path.expanduser("~"), ".workflow_config", "launchpad", name + "_launchpad.yaml")) if config == "fworker": return FWorker.from_file( os.path.join(os.path.expanduser("~"), ".workflow_config", "fworker", name + "_fworker.yaml")) if config == "qadapter": return CommonAdapter.from_file( os.path.join(os.path.expanduser("~"), ".workflow_config", "fworker", name + "_qadapter.yaml")) except FileNotFoundError: raise FileNotFoundError( "Did not find the corresponding configuration file in " + os.path.join(os.path.expanduser("~"), ".workflow_config") + ". Use 'vsc config " + config + "' to set up the " + name + " configuration for the " + config + ".")
val_spec = copy.deepcopy(spec) val_spec['TRAINING_SEQUENCES'] = VALIDATION_SEQUENCES val_spec['seq_idx_to_eval'] = VALIDATION_SEQUENCES val_spec['validation_eval'] = True val_batch = Firework(RunRBPF_Batch(), spec = val_spec) val_eval = Firework(RunEval(), spec = val_spec) storeResultsFW2 = Firework(StoreResultsInDatabase(), spec=val_spec) first_iter= Firework(Iterate(), spec = spec) workflow = Workflow([init_batch, eval_init, first_iter, val_batch, val_eval, storeResultsFW1, storeResultsFW2], {init_batch: [eval_init], eval_init: [first_iter], val_batch: [val_eval], eval_init:[storeResultsFW1], val_eval:[storeResultsFW2]}) launchpad.add_wf(workflow) qadapter = CommonAdapter.from_file("%sfireworks_files/my_qadapter.yaml" % RBPF_HOME_DIRECTORY) # rapidfire(launchpad, FWorker(), qadapter, launch_dir='.', nlaunches='infinite', njobs_queue=20, # njobs_block=500, sleep_time=None, reserve=False, strm_lvl='INFO', timeout=None, # fill_mode=False) fworker = FWorker() rapidfire(launchpad, fworker, qadapter, launch_dir='.', nlaunches='infinite', njobs_queue=20, njobs_block=500, sleep_time=None, reserve=False, strm_lvl="DEBUG", timeout=None, fill_mode=False)