Пример #1
0
def grid_jug(jug_args, jug_nworkers=4, name='gridjug', **kwargs):
    """
    A light-weight wrapper to run jug with gridmap on a Grid Engine cluster
    """

    from gridmap.job import grid_map

    jug_argv = ['dummy_script_name_not_parsed_by_jug', 'execute']
    jug_argv.extend(jug_args)

    # function arguments for grid_map
    # note that there are multiple lists here
    # the innermost list is the list of arguments to jug
    # this needs to stay a list as jug.main accepts a single argument argv
    # which is a list of parameters for jug
    # https://github.com/luispedro/jug/blob/15a7043f6f859810b5e6af1638176d1a9cb70f5a/jug/jug.py#L405
    #
    # we wrap this inner list in a wrapper list [jug_argv] that gridmap
    # "expands" to its single item jug_argv upon calling the jug.main function,
    # with that very single item jug_argv as the single argument
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L225
    #
    # finally, the wrapper list [jug_arvg] is contained in the outer list
    # [[jug_argv]]. The outer list is multiplied by the number of workers to
    # create an outer list of that many items, each of which is the wrapped
    # list [jug_argv] to supplied to each of the worker jobs
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L929
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L933
    #
    args_list = jug_nworkers * [[jug_argv]]

    return grid_map(f=_jug_main, args_list=args_list, name=name, **kwargs)
Пример #2
0
def grid_jug(jug_args, jug_nworkers=4, name='gridjug', **kwargs):
    """
    A light-weight wrapper to run jug with gridmap on a Grid Engine cluster
    """

    from gridmap.job import grid_map
    
    jug_argv = ['dummy_script_name_not_parsed_by_jug', 'execute']
    jug_argv.extend(jug_args)

    # function arguments for grid_map
    # note that there are multiple lists here
    # the innermost list is the list of arguments to jug
    # this needs to stay a list as jug.main accepts a single argument argv
    # which is a list of parameters for jug
    # https://github.com/luispedro/jug/blob/15a7043f6f859810b5e6af1638176d1a9cb70f5a/jug/jug.py#L405
    #
    # we wrap this inner list in a wrapper list [jug_argv] that gridmap
    # "expands" to its single item jug_argv upon calling the jug.main function,
    # with that very single item jug_argv as the single argument
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L225
    #
    # finally, the wrapper list [jug_arvg] is contained in the outer list
    # [[jug_argv]]. The outer list is multiplied by the number of workers to
    # create an outer list of that many items, each of which is the wrapped
    # list [jug_argv] to supplied to each of the worker jobs
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L929
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L933
    #
    args_list = jug_nworkers * [[jug_argv]]

    return grid_map(
        f=_jug_main,
        args_list=args_list,
        name=name,
        **kwargs
    )
Пример #3
0
Файл: grid.py Проект: andsor/jug
def grid_jug(jug_args, jug_nworkers=4, name="gridjug", **kwargs):
    """
    A light-weight wrapper to run Jug with GridMap on a Grid Engine cluster

    From their own description, GridMap is a package that allows to easily
    create jobs on a Grid Engine powered cluster directly from Python.
    This wrapper lets GridMap simply spawn several jug-execute workers on a
    Grid Engine cluster.
    Thus we have the benefit of programmatic (reproducible) execution of Jug
    processes.
    Furthermore, GridMap adds a convenient monitoring and reporting layer.
    Under the hood, of course, Jug keeps doing the actual work.

    Parameters
    ----------

    jug_args : list
        Jug command-line arguments.
        Note that ``'execute'`` is already included.
        The command line is roughly equivalent to:

            'jug execute ' + ' '.join(jug_args)

    jug_nworkers : int, optional
        number of Grid Engine tasks to start
        (i.e. number of times 'jug execute' is run)

    name : str, optional
        base name of the Grid Engine task

    **kwargs : keyword-dict, optional
        additional options passed through to gridmap.grid_map

    See Also
    --------

    `GridMap <https://github.com/pygridtools/gridmap>`_

    `gridmap.grid_map <http://gridmap.readthedocs.org/en/latest/gridmap.html#gridmap.job.grid_map>`_
    """

    from gridmap.job import grid_map

    jug_argv = ["jug", "execute"]
    jug_argv.extend(jug_args)

    # function arguments for grid_map
    # note that there are multiple lists here
    # the innermost list is the list of arguments to jug
    # this needs to stay a list as jug.main accepts a single argument argv
    # which is a list of parameters for jug
    # https://github.com/luispedro/jug/blob/15a7043f6f859810b5e6af1638176d1a9cb70f5a/jug/jug.py#L405
    #
    # we wrap this inner list in a wrapper list [jug_argv] that gridmap
    # "expands" to its single item jug_argv upon calling the jug.main function,
    # with that very single item jug_argv as the single argument
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L225
    #
    # finally, the wrapper list [jug_arvg] is contained in the outer list
    # [[jug_argv]]. The outer list is multiplied by the number of workers to
    # create an outer list of that many items, each of which is the wrapped
    # list [jug_argv] to supplied to each of the worker jobs
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L929
    # https://github.com/pygridtools/gridmap/blob/master/gridmap/job.py#L933
    #
    args_list = jug_nworkers * [[jug_argv]]

    return grid_map(f=_jug_main, args_list=args_list, name=name, **kwargs)