def get_cluster(target,
                numprocesses=5,
                frontend=DEFAULT_FRONTEND,
                backend=DEFAULT_BACKEND,
                heartbeat=DEFAULT_HEARTBEAT,
                register=DEFAULT_REG,
                working_dir='.',
                logfile='stdout',
                debug=False,
                background=False,
                worker_params=None,
                timeout=DEFAULT_TIMEOUT_MOVF,
                max_age=DEFAULT_MAX_AGE,
                max_age_delta=DEFAULT_MAX_AGE_DELTA):
    """Runs a Powerhose cluster.

    Options:

    - **callable**: The Python callable that will be called when the broker
      receive a job.
    - **numprocesses**: The number of workers. Defaults to 5.
    - **frontend**: the ZMQ socket to receive jobs.
    - **backend**: the ZMQ socket to communicate with workers.
    - **register** : the ZMQ socket to register workers
    - **heartbeat**: the ZMQ socket to receive heartbeat requests
    - **working_dir**: The working directory. Defaults to *"."*
    - **logfile**: The file to log into. Defaults to stdout.
    - **debug**: If True, the logs are at the DEBUG level. Defaults to False
    - **background**: If True, the cluster is run in the background.
      Defaults to False.
    - **worker_params**: a dict of params to pass to the worker. Default is
      None
    - **timeout** the maximum time allowed before the thread stacks is dumped
      and the job result not sent back.
    - **max_age**: maximum age for a worker in seconds. After that delay,
      the worker will simply quit. When set to -1, never quits.
      Defaults to -1.
    - **max_age_delta**: maximum value in seconds added to max age.
      The Worker will quit after *max_age + random(0, max_age_delta)*
      This is done to avoid having all workers quit at the same instant.
    """
    from circus import get_arbiter

    python = sys.executable
    if debug:
        debug = ' --debug'
    else:
        debug = ''
    if worker_params:
        params = encode_params(worker_params)

    broker_cmd = [
        python, '-m', 'powerhose.broker', '--logfile', logfile, debug,
        '--frontend', frontend, '--backend', backend, '--heartbeat', heartbeat
    ]

    worker_cmd = [
        python, '-m', 'powerhose.worker', target, '--logfile', logfile, debug,
        '--backend', backend, '--heartbeat', heartbeat, '--timeout',
        str(timeout), '--max-age',
        str(max_age), '--max-age-delta',
        str(max_age_delta)
    ]

    if worker_params:
        worker_cmd += ['--params', params]

    if logfile == 'stdout':
        stream = {'class': 'StdoutStream'}
    else:
        stream = {'class': 'FileStream', 'filename': logfile}

    watchers = [{
        'name': 'broker',
        'cmd': ' '.join(broker_cmd),
        'working_dir': working_dir,
        'executable': python,
        'stderr_stream': stream,
        'stdout_stream': stream
    }, {
        'name': 'workers',
        'cmd': ' '.join(worker_cmd),
        'numprocesses': numprocesses,
        'working_dir': working_dir,
        'executable': python,
        'stderr_stream': stream,
        'stdout_stream': stream
    }]

    # XXX add more options
    arbiter = get_arbiter(watchers, background=background)

    # give a chance to all processes to start
    # XXX this should be in Circus
    if background:
        start = time.clock()
        while time.clock() - start < 5:
            statuses = [
                status == 'active' for status in arbiter.statuses().values()
            ]
            if all(statuses):
                break

    return arbiter
Exemplo n.º 2
0
def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND,
                backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT,
                register=DEFAULT_REG,
                working_dir='.', logfile='stdout',
                debug=False, background=False, worker_params=None,
                timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE,
                max_age_delta=DEFAULT_MAX_AGE_DELTA):
    """Runs a Powerhose cluster.

    Options:

    - **callable**: The Python callable that will be called when the broker
      receive a job.
    - **numprocesses**: The number of workers. Defaults to 5.
    - **frontend**: the ZMQ socket to receive jobs.
    - **backend**: the ZMQ socket to communicate with workers.
    - **register** : the ZMQ socket to register workers
    - **heartbeat**: the ZMQ socket to receive heartbeat requests
    - **working_dir**: The working directory. Defaults to *"."*
    - **logfile**: The file to log into. Defaults to stdout.
    - **debug**: If True, the logs are at the DEBUG level. Defaults to False
    - **background**: If True, the cluster is run in the background.
      Defaults to False.
    - **worker_params**: a dict of params to pass to the worker. Default is
      None
    - **timeout** the maximum time allowed before the thread stacks is dumped
      and the job result not sent back.
    - **max_age**: maximum age for a worker in seconds. After that delay,
      the worker will simply quit. When set to -1, never quits.
      Defaults to -1.
    - **max_age_delta**: maximum value in seconds added to max age.
      The Worker will quit after *max_age + random(0, max_age_delta)*
      This is done to avoid having all workers quit at the same instant.
    """
    from circus import get_arbiter

    python = sys.executable
    if debug:
        debug = ' --debug'
    else:
        debug = ''
    if worker_params:
        params = encode_params(worker_params)

    broker_cmd = [python, '-m', 'powerhose.broker', '--logfile',  logfile,
                  debug, '--frontend', frontend, '--backend', backend,
                  '--heartbeat', heartbeat]

    worker_cmd = [python, '-m', 'powerhose.worker', target, '--logfile',
                  logfile, debug, '--backend', backend, '--heartbeat',
                  heartbeat, '--timeout', str(timeout), '--max-age',
                  str(max_age), '--max-age-delta', str(max_age_delta)]

    if worker_params:
        worker_cmd += ['--params', params]

    if logfile == 'stdout':
        stream = {'class': 'StdoutStream'}
    else:
        stream = {'class': 'FileStream',
                  'filename': logfile}

    watchers = [{'name': 'broker',
                 'cmd': ' '.join(broker_cmd),
                 'working_dir': working_dir,
                 'executable': python,
                 'stderr_stream': stream,
                 'stdout_stream': stream
                 },
                {'name': 'workers',
                 'cmd': ' '.join(worker_cmd),
                 'numprocesses': numprocesses,
                 'working_dir': working_dir,
                 'executable': python,
                 'stderr_stream': stream,
                 'stdout_stream': stream
                 }
                ]

    # XXX add more options
    arbiter = get_arbiter(watchers, background=background)

    # give a chance to all processes to start
    # XXX this should be in Circus
    if background:
        start = time.clock()
        while time.clock() - start < 5:
            statuses = [status == 'active' for status in
                        arbiter.statuses().values()]
            if all(statuses):
                break

    return arbiter
Exemplo n.º 3
0
def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND,
                backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT,
                working_dir='.', logfile='stdout',
                debug=False, background=False, worker_params=None,
                timeout=DEFAULT_TIMEOUT_MOVF):
    """Runs a Powerhose cluster.

    Options:

    - **callable**: The Python callable that will be called when the broker
      receive a job.
    - **numprocesses**: The number of workers. Defaults to 5.
    - **frontend**: the ZMQ socket to receive jobs.
    - **backend**: the ZMQ socket to communicate with workers.
    - **heartbeat**: the ZMQ socket to receive heartbeat requests/
    - **working_dir**: The working directory. Defaults to *"."*
    - **logfile**: The file to log into. Defaults to stdout.
    - **debug**: If True, the logs are at the DEBUG level. Defaults to False
    - **background**: If True, the cluster is run in the background.
      Defaults to False.
    - **worker_params**: a dict of params to pass to the worker. Default is
      None
    - **timeout** the maximum time allowed before the thread stacks is dumped
      and the job result not sent back.
    """
    from circus import get_arbiter
    from circus.stream import StdoutStream, FileStream

    python = sys.executable
    if debug:
        debug = ' --debug'
    else:
        debug = ''
    if worker_params:
        params = encode_params(worker_params)

    broker_cmd = [python, '-m', 'powerhose.broker', '--logfile',  logfile,
                  debug, '--frontend', frontend, '--backend', backend,
                  '--heartbeat', heartbeat]

    worker_cmd = [python, '-m', 'powerhose.worker', target, '--logfile',
                  logfile, debug, '--backend', backend, '--heartbeat',
                  heartbeat, '--timeout', str(timeout)]

    if worker_params:
        worker_cmd += ['--params', params]

    if logfile == 'stdout':
        stream = {'class': StdoutStream}
    else:
        stream = {'class': FileStream}

    watchers = [{'name': 'broker',
                 'cmd': ' '.join(broker_cmd),
                 'working_dir': working_dir,
                 'executable': python,
                 'stderr_stream': stream,
                 'stdout_stream': stream
                 },
                {'name': 'workers',
                 'cmd': ' '.join(worker_cmd),
                 'numprocesses': numprocesses,
                 'working_dir': working_dir,
                 'executable': python,
                 'stderr_stream': stream,
                 'stdout_stream': stream

                 }
                ]

    # XXX add more options
    arbiter = get_arbiter(watchers, background=background)

    # give a chance to all processes to start
    # XXX this should be in Circus
    if background:
        start = time.clock()
        while time.clock() - start < 5:
            statuses = [status == 'active' for status in
                        arbiter.statuses().values()]
            if all(statuses):
                break

    return arbiter