def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, working_dir='.', logfile='stdout', debug=False, background=False, worker_params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): """Runs a Powerhose cluster. Options: - **callable**: The Python callable that will be called when the broker receive a job. - **numprocesses**: The number of workers. Defaults to 5. - **frontend**: the ZMQ socket to receive jobs. - **backend**: the ZMQ socket to communicate with workers. - **register** : the ZMQ socket to register workers - **heartbeat**: the ZMQ socket to receive heartbeat requests - **working_dir**: The working directory. Defaults to *"."* - **logfile**: The file to log into. Defaults to stdout. - **debug**: If True, the logs are at the DEBUG level. Defaults to False - **background**: If True, the cluster is run in the background. Defaults to False. - **worker_params**: a dict of params to pass to the worker. Default is None - **timeout** the maximum time allowed before the thread stacks is dumped and the job result not sent back. - **max_age**: maximum age for a worker in seconds. After that delay, the worker will simply quit. When set to -1, never quits. Defaults to -1. - **max_age_delta**: maximum value in seconds added to max age. The Worker will quit after *max_age + random(0, max_age_delta)* This is done to avoid having all workers quit at the same instant. """ from circus import get_arbiter python = sys.executable if debug: debug = ' --debug' else: debug = '' if worker_params: params = encode_params(worker_params) broker_cmd = [ python, '-m', 'powerhose.broker', '--logfile', logfile, debug, '--frontend', frontend, '--backend', backend, '--heartbeat', heartbeat ] worker_cmd = [ python, '-m', 'powerhose.worker', target, '--logfile', logfile, debug, '--backend', backend, '--heartbeat', heartbeat, '--timeout', str(timeout), '--max-age', str(max_age), '--max-age-delta', str(max_age_delta) ] if worker_params: worker_cmd += ['--params', params] if logfile == 'stdout': stream = {'class': 'StdoutStream'} else: stream = {'class': 'FileStream', 'filename': logfile} watchers = [{ 'name': 'broker', 'cmd': ' '.join(broker_cmd), 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }, { 'name': 'workers', 'cmd': ' '.join(worker_cmd), 'numprocesses': numprocesses, 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }] # XXX add more options arbiter = get_arbiter(watchers, background=background) # give a chance to all processes to start # XXX this should be in Circus if background: start = time.clock() while time.clock() - start < 5: statuses = [ status == 'active' for status in arbiter.statuses().values() ] if all(statuses): break return arbiter
def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, working_dir='.', logfile='stdout', debug=False, background=False, worker_params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): """Runs a Powerhose cluster. Options: - **callable**: The Python callable that will be called when the broker receive a job. - **numprocesses**: The number of workers. Defaults to 5. - **frontend**: the ZMQ socket to receive jobs. - **backend**: the ZMQ socket to communicate with workers. - **register** : the ZMQ socket to register workers - **heartbeat**: the ZMQ socket to receive heartbeat requests - **working_dir**: The working directory. Defaults to *"."* - **logfile**: The file to log into. Defaults to stdout. - **debug**: If True, the logs are at the DEBUG level. Defaults to False - **background**: If True, the cluster is run in the background. Defaults to False. - **worker_params**: a dict of params to pass to the worker. Default is None - **timeout** the maximum time allowed before the thread stacks is dumped and the job result not sent back. - **max_age**: maximum age for a worker in seconds. After that delay, the worker will simply quit. When set to -1, never quits. Defaults to -1. - **max_age_delta**: maximum value in seconds added to max age. The Worker will quit after *max_age + random(0, max_age_delta)* This is done to avoid having all workers quit at the same instant. """ from circus import get_arbiter python = sys.executable if debug: debug = ' --debug' else: debug = '' if worker_params: params = encode_params(worker_params) broker_cmd = [python, '-m', 'powerhose.broker', '--logfile', logfile, debug, '--frontend', frontend, '--backend', backend, '--heartbeat', heartbeat] worker_cmd = [python, '-m', 'powerhose.worker', target, '--logfile', logfile, debug, '--backend', backend, '--heartbeat', heartbeat, '--timeout', str(timeout), '--max-age', str(max_age), '--max-age-delta', str(max_age_delta)] if worker_params: worker_cmd += ['--params', params] if logfile == 'stdout': stream = {'class': 'StdoutStream'} else: stream = {'class': 'FileStream', 'filename': logfile} watchers = [{'name': 'broker', 'cmd': ' '.join(broker_cmd), 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }, {'name': 'workers', 'cmd': ' '.join(worker_cmd), 'numprocesses': numprocesses, 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream } ] # XXX add more options arbiter = get_arbiter(watchers, background=background) # give a chance to all processes to start # XXX this should be in Circus if background: start = time.clock() while time.clock() - start < 5: statuses = [status == 'active' for status in arbiter.statuses().values()] if all(statuses): break return arbiter
def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, working_dir='.', logfile='stdout', debug=False, background=False, worker_params=None, timeout=DEFAULT_TIMEOUT_MOVF): """Runs a Powerhose cluster. Options: - **callable**: The Python callable that will be called when the broker receive a job. - **numprocesses**: The number of workers. Defaults to 5. - **frontend**: the ZMQ socket to receive jobs. - **backend**: the ZMQ socket to communicate with workers. - **heartbeat**: the ZMQ socket to receive heartbeat requests/ - **working_dir**: The working directory. Defaults to *"."* - **logfile**: The file to log into. Defaults to stdout. - **debug**: If True, the logs are at the DEBUG level. Defaults to False - **background**: If True, the cluster is run in the background. Defaults to False. - **worker_params**: a dict of params to pass to the worker. Default is None - **timeout** the maximum time allowed before the thread stacks is dumped and the job result not sent back. """ from circus import get_arbiter from circus.stream import StdoutStream, FileStream python = sys.executable if debug: debug = ' --debug' else: debug = '' if worker_params: params = encode_params(worker_params) broker_cmd = [python, '-m', 'powerhose.broker', '--logfile', logfile, debug, '--frontend', frontend, '--backend', backend, '--heartbeat', heartbeat] worker_cmd = [python, '-m', 'powerhose.worker', target, '--logfile', logfile, debug, '--backend', backend, '--heartbeat', heartbeat, '--timeout', str(timeout)] if worker_params: worker_cmd += ['--params', params] if logfile == 'stdout': stream = {'class': StdoutStream} else: stream = {'class': FileStream} watchers = [{'name': 'broker', 'cmd': ' '.join(broker_cmd), 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }, {'name': 'workers', 'cmd': ' '.join(worker_cmd), 'numprocesses': numprocesses, 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream } ] # XXX add more options arbiter = get_arbiter(watchers, background=background) # give a chance to all processes to start # XXX this should be in Circus if background: start = time.clock() while time.clock() - start < 5: statuses = [status == 'active' for status in arbiter.statuses().values()] if all(statuses): break return arbiter