Beispiel #1
0
def qadapter(qadapter_file=None, fworker_name="base"):
    """
    Script to set up the configuration of the queueing system. Note that we store
    the queue_adapter in the same configuration directory as the Fireworker,
    i.e. fworker. This is in the assumption that each worker has one preferred
    queueing system.

    Args:
        qadapter_file (str): my_qadapter_file.yaml from which to configure the
            queue adapter.
        fworker_name (str): Which fworker configuration to add the queue adapter
            to. Note that there has to be a corresponding job_template.sh file for
            the queue adapter to adjust before submitting it to the fireworker queue.

    Returns:
        None

    """
    if qadapter_file:
        queue_adapter = CommonAdapter.from_file(qadapter_file)
    else:
        logdir = input(
            "Please provide the path to the directory where the log "
            "should be stored. \n"
            "(Note: if the directory does not exist, it will be created): ")
        logdir = os.path.abspath(logdir)
        if not os.path.exists(logdir):
            os.makedirs(logdir)
        queue_adapter = CommonAdapter.from_file(
            os.path.join(os.path.dirname(os.path.abspath(__file__)),
                         "examples", "config", "fworker",
                         "example_qadapter.yaml"))
        queue_adapter["logdir"] = logdir
        print(
            "\nNote: 'rocket_launch' has been set to an infinite rapidfire mode."
        )

    try:
        os.makedirs(
            os.path.join(os.path.expanduser("~"), ".workflow_config",
                         "fworker"))
    except FileExistsError:
        pass

    config_q_file = os.path.join(os.path.expanduser("~"), ".workflow_config",
                                 "fworker", fworker_name + "_qadapter.yaml")

    template_file = os.path.join(os.path.expanduser("~"), ".workflow_config",
                                 "fworker", fworker_name + "_job_template.sh")
    if not os.path.exists(template_file):
        print()
        warn_message = "No corresponding template file found! Don't forget to use " \
                       "'vsc config jobscript' to add the template file of the '" \
                       + fworker_name + "' fireworker."
        warnings.warn(warn_message)

    queue_adapter.template_file = template_file
    queue_adapter.to_file(config_q_file)
    print("\nQueue adapter file written to " + config_q_file)
Beispiel #2
0
def load_config(config, name="base"):
    """
    Load a LaunchPad, FWorker of QueueAdapter from the configuration files.

    Args:
        config (str): Type of configuration file to load. Either "launchpad", "fworker"
            or "qadapter".
        name (str): Name of the configuration. Defaults to "base".

    Returns:
        Either a LaunchPad, FWorker or QueuAdapter, depending on the "config" argument.

    """
    try:
        if config == "launchpad":
            return LaunchPad.from_file(
                os.path.join(os.path.expanduser("~"), ".workflow_config",
                             "launchpad", name + "_launchpad.yaml"))
        if config == "fworker":
            return FWorker.from_file(
                os.path.join(os.path.expanduser("~"), ".workflow_config",
                             "fworker", name + "_fworker.yaml"))
        if config == "qadapter":
            return CommonAdapter.from_file(
                os.path.join(os.path.expanduser("~"), ".workflow_config",
                             "fworker", name + "_qadapter.yaml"))
    except FileNotFoundError:
        raise FileNotFoundError(
            "Did not find the corresponding configuration file in " +
            os.path.join(os.path.expanduser("~"), ".workflow_config") +
            ". Use 'vsc config " + config + "' to set up the " + name +
            " configuration for the " + config + ".")

    val_spec = copy.deepcopy(spec)
    val_spec['TRAINING_SEQUENCES'] = VALIDATION_SEQUENCES    
    val_spec['seq_idx_to_eval'] = VALIDATION_SEQUENCES         
    val_spec['validation_eval'] = True   
    val_batch = Firework(RunRBPF_Batch(), spec = val_spec)    
    val_eval = Firework(RunEval(), spec = val_spec)
    storeResultsFW2 = Firework(StoreResultsInDatabase(), spec=val_spec)

    first_iter= Firework(Iterate(), spec = spec)
    workflow = Workflow([init_batch, eval_init, first_iter, val_batch, val_eval, storeResultsFW1, storeResultsFW2], 
                        {init_batch: [eval_init], eval_init: [first_iter], val_batch: [val_eval], eval_init:[storeResultsFW1], val_eval:[storeResultsFW2]})

    launchpad.add_wf(workflow)
    qadapter = CommonAdapter.from_file("%sfireworks_files/my_qadapter.yaml" % RBPF_HOME_DIRECTORY)
#    rapidfire(launchpad, FWorker(), qadapter, launch_dir='.', nlaunches='infinite', njobs_queue=20,
#                  njobs_block=500, sleep_time=None, reserve=False, strm_lvl='INFO', timeout=None,
#                  fill_mode=False)


    fworker = FWorker()
    rapidfire(launchpad, fworker, qadapter, launch_dir='.', nlaunches='infinite', njobs_queue=20,
      njobs_block=500, sleep_time=None, reserve=False, strm_lvl="DEBUG", timeout=None,
      fill_mode=False)