Example #1
0
def create_job(build_command,
               easyconfig,
               output_dir=None,
               conn=None,
               ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    @param build_command: format string for command, full path to an easyconfig file will be substituted in it
    @param easyconfig: easyconfig as processed by process_easyconfig
    @param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
    @param conn: open connection to PBS server
    @param ppn: ppn setting to use (# 'processors' (cores) per node to use)
    returns the job
    """
    if output_dir is None:
        output_dir = 'easybuild-build'

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" %
              easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'],
                det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    # create command based on build_command template
    command = build_command % {
        'spec': easyconfig['spec'],
        'output_dir': os.path.join(os.path.abspath(output_dir), name),
    }

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command,
                 name,
                 easybuild_vars,
                 resources=resources,
                 conn=conn,
                 ppn=ppn)
    job.module = easyconfig['ec'].full_mod_name

    return job
Example #2
0
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {'spec': easyconfig['spec']}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" %
              easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'],
                det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    var = config.oldstyle_environment_variables['test_output_path']
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command,
                 name,
                 easybuild_vars,
                 resources=resources,
                 conn=conn,
                 ppn=ppn)
    job.module = det_full_module_name(easyconfig['ec'])

    return job
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    @param build_command: format string for command, full path to an easyconfig file will be substituted in it
    @param easyconfig: easyconfig as processed by process_easyconfig
    @param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
    @param conn: open connection to PBS server
    @param ppn: ppn setting to use (# 'processors' (cores) per node to use)
    returns the job
    """
    if output_dir is None:
        output_dir = 'easybuild-build'

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    # create command based on build_command template
    command = build_command % {
        'spec': easyconfig['spec'],
        'output_dir': os.path.join(os.path.abspath(output_dir), name),
    }

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = easyconfig['ec'].full_mod_name

    return job
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {'spec': easyconfig['spec']}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    var = config.oldstyle_environment_variables['test_output_path']
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = det_full_module_name(easyconfig['ec'])

    return job
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {"spec": easyconfig["spec"]}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # create unique name based on module name
    name = "%s-%s" % easyconfig["module"]

    var = config.oldstyle_environment_variables["test_output_path"]
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    buildstats = get_repository().get_buildstats(*easyconfig["module"])
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]["build_time"]
        resources["hours"] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = easyconfig["module"]

    return job
Example #6
0
def create_job(build_command, package, output_dir=""):
    """
    Creates a job, to build a *single* package
    build_command is a format string in which a full path to an eb file will be substituted
    package should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % package['spec']

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    # create unique name based on module name
    name = "%s-%s" % package['module']

    easybuild_vars['EASYBUILDTESTOUTPUT'] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    buildstats = getRepository().get_buildstats(*package['module'])
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources)
    job.module = package['module']

    return job