def _create_circus(cls, callable, plugins=None, stats=False, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = { 'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test', 'graceful_timeout': 4 } worker.update(kw) debug = kw.get('debug', False) if stats: arbiter = get_arbiter([worker], background=True, plugins=plugins, stats_endpoint=DEFAULT_ENDPOINT_STATS, statsd=True, debug=debug, statsd_close_outputs=not debug) else: arbiter = get_arbiter([worker], background=True, plugins=plugins, debug=debug) arbiter.start() return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = { 'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test', 'graceful_timeout': 4 } worker.update(kw) if stats: arbiter = get_arbiter([worker], background=True, plugins=plugins, stats_endpoint=DEFAULT_ENDPOINT_STATS, debug=kw.get('debug', False)) else: arbiter = get_arbiter([worker], background=True, plugins=plugins, debug=kw.get('debug', False)) arbiter.start() time.sleep(.3) self.arbiters.append(arbiter) self.files.append(testfile) return testfile
def start_manager(self): if settings.SETTINGS_TYPE == 'production': arbiter = get_arbiter([{"cmd": "python " "" + settings.ENVIRONMENTS_MANAGER_PATH, "numprocesses": 1}], background=True) if check_python_process(): clean_environments() arbiter.start()
def _create_circus(cls, callable, plugins=None, stats=False, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test', 'graceful_timeout': 4} worker.update(kw) if stats: arbiter = get_arbiter([worker], background=True, plugins=plugins, stats_endpoint=DEFAULT_ENDPOINT_STATS, debug=kw.get('debug', False)) else: arbiter = get_arbiter([worker], background=True, plugins=plugins, debug=kw.get('debug', False)) arbiter.start() return testfile, arbiter
def start_manager(self): arbiter = get_arbiter([{ "cmd": "python " "" + settings.ENVIRONMENTS_MANAGER_PATH, "numprocesses": 1 }], background=True) if settings.SETTINGS_TYPE == 'production': arbiter.start() self.started = True
def _create_circus(cls, callable, plugins=None, stats=False, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ["generic.py", callable, testfile] worker = {"cmd": _CMD, "args": args, "working_dir": wdir, "name": "test", "graceful_timeout": 4} worker.update(kw) if stats: arbiter = get_arbiter( [worker], background=True, plugins=plugins, stats_endpoint=DEFAULT_ENDPOINT_STATS, debug=kw.get("debug", False), ) else: arbiter = get_arbiter([worker], background=True, plugins=plugins, debug=kw.get("debug", False)) arbiter.start() return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test'} worker.update(kw) if stats: arbiter = get_arbiter([worker], background=True, plugins=plugins, stats_endpoint='tcp://127.0.0.1:5557') else: arbiter = get_arbiter([worker], background=True, plugins=plugins) arbiter.start() time.sleep(.3) self.arbiters.append(arbiter) self.files.append(testfile) return testfile
def _run_circus(self, callable, plugins=None, stats=False, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test', 'graceful_timeout': 4} worker.update(kw) if stats: arbiter = get_arbiter([worker], background=True, plugins=plugins, stats_endpoint='tcp://127.0.0.1:5557') else: arbiter = get_arbiter([worker], background=True, plugins=plugins) arbiter.start() time.sleep(.3) self.arbiters.append(arbiter) self.files.append(testfile) return testfile
def _run_circus(self, callable): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) cmd = '%s generic.py %s %s' % (sys.executable, callable, testfile) worker = {'cmd': cmd, 'working_dir': wdir, 'name': 'test'} arbiter = get_arbiter([worker], background=True) arbiter.start() self.arbiters.append(arbiter) self.files.append(testfile) return testfile
def setUpClass(cls): settings = {} load_into_settings(CONFIG_PATH, settings) cls.config = settings['config'] cls.arbiter = circus.get_arbiter([ {'cmd': 'python router.py %s' % CONFIG_PATH, 'shell': True} ], background=True) cls.arbiter.start() # I hope the router is set up once we're done sleeping! time.sleep(1)
def __init__(self, host, port, timeout=15): assert type(host) == str assert type(port) == int and port >= 0 and port <= 65535 assert type(timeout) == int and timeout > 0 self._host = host self._port = port self._timeout = timeout self._arbiter = get_arbiter([]) self._arbiter.start() self._client = CircusClient(timeout=self._timeout, endpoint='tcp://{0}:{1}'.format( self._host, self._port))
def __init__(self, host, port, timeout=15): assert type(host) == str assert type(port) == int and port >= 0 and port <= 65535 assert type(timeout) == int and timeout > 0 self._host = host self._port = port self._timeout = timeout self._arbiter = get_arbiter([]) self._arbiter.start() self._client = CircusClient(timeout=self._timeout, endpoint='tcp://{0}:{1}'.format(self._host, self._port))
def _run_circus(self, callable): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) cmd = "%s generic.py %s %s" % (sys.executable, callable, testfile) arbiter = get_arbiter(cmd, working_dir=wdir, numprocesses=1, name="test") runner = Runner(arbiter, testfile) runner.start() self.runners.append(runner) self.files.append(testfile) return testfile
def _run_circus(self, callable): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ["generic.py", callable, testfile] worker = {"cmd": _CMD, "args": args, "working_dir": wdir, "name": "test"} arbiter = get_arbiter([worker], background=True) arbiter.start() time.sleep(0.3) self.arbiters.append(arbiter) self.files.append(testfile) return testfile
def worker(): cwd = os.path.abspath('.') program = { 'name': 'worker', 'use': 'circus.plugins.redis_observer.RedisObserver', 'loop_rate': 5, 'cmd': '.venv/bin/python worker.py', 'working_dir': cwd, 'sample_rate': 2.0, 'application_name': 'eclogue-worker', } arbiter = get_arbiter(watchers=[program]) arbiter.start()
def work(n): """ Spin up workers. """ workers = {"cmd": os.path.join(config["osp"]["bin"], "rqworker"), "numprocesses": n, "env": {"LANG": "en_US.UTF-8"}} arbiter = get_arbiter([workers]) try: arbiter.start() finally: arbiter.stop()
def onboard(ctx, rover='hughey'): """ Launches a supervisor for the onboard software stack. (Defaults to Hughey) """ arbiter = get_arbiter({ 'cmd': p, 'numprocesses': 1, 'virtualenv': config.PRODUCT_ENV, 'copy_env': True } for p in ONBOARD_PROGRAMS) try: arbiter.start() finally: arbiter.stop()
def _run_circus(self, callable, plugins=None, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test'} worker.update(kw) arbiter = get_arbiter([worker], background=True, plugins=plugins) arbiter.start() time.sleep(.3) self.arbiters.append(arbiter) self.files.append(testfile) return testfile
def work(n): """ Spin up workers. """ workers = { 'cmd': os.path.join(config['osp']['bin'], 'rqworker'), 'numprocesses': n, 'env': {'LANG': 'en_US.UTF-8'} } arbiter = get_arbiter([workers]) try: arbiter.start() finally: arbiter.stop()
def work(n): """ Spin up workers. """ workers = { 'cmd': os.path.join(config['osp']['bin'], 'rqworker'), 'numprocesses': n, 'env': { 'LANG': 'en_US.UTF-8' } } arbiter = get_arbiter([workers]) try: arbiter.start() finally: arbiter.stop()
def _run_circus(self, callable, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = { 'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test' } worker.update(kw) arbiter = get_arbiter([worker], background=True) arbiter.start() time.sleep(.3) self.arbiters.append(arbiter) self.files.append(testfile) return testfile
def start( exe_path: Path = DEFAULT_INSTALL_PATH, data_path: Path = DEFAULT_DATA_PATH, connection_uri: str = DEFAULT_CONNECTION_URI, database: str = SYSTEM_DATABASE, username: str = DEFAULT_USERNAME, password: str = DEFAULT_PASSWORD, close_stdout_and_stderr: bool = False, ) -> Arbiter: if exe_path.name != "arangodb": exe_path = get_exe_path(exe_path) working_dir_path = exe_path.parent env_path = f"{os.environ['PATH']}:{working_dir_path}" arbiter = get_arbiter([{ "cmd": f"{exe_path} --starter.mode single --starter.data-dir {data_path}", "working_dir": working_dir_path, "env": { "PATH": env_path }, "close_child_stdout": close_stdout_and_stderr, "close_child_stderr": close_stdout_and_stderr, }], background=True) arbiter.start() while not is_running(connection_uri=connection_uri, database=database, username=username, password=password): time.sleep(START_SLEEP_DELAY) return arbiter
def start(): daemonize() # ~ arbiter = get_arbiter([get_streamer('bla')], # ~ controller='tcp://127.0.0.1:6000', # ~ logoutput='arbiter.log', loglevel='INFO', debug=False, statsd=True) arbiter = get_arbiter([get_daemon_properties('ricoh')], controller='tcp://127.0.0.1:6000', logoutput='arbiter.log', loglevel='INFO', debug=False, statsd=True) restart = True while restart: try: future = arbiter.start() restart = False if check_future_exception_and_log(future) is None: restart = arbiter._restarting except Exception as e: # emergency stop arbiter.loop.run_sync(arbiter._emergency_stop) raise(e) except KeyboardInterrupt: pass finally: arbiter = None sys.exit(0)
def start_manager(self): if settings.SETTINGS_TYPE == 'production': arbiter = get_arbiter([{"cmd": "python " "" + settings.ENVIRONMENTS_MANAGER_PATH, "numprocesses": 1}], background=True) arbiter.start() self.started = True
def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, working_dir='.', logfile='stdout', debug=False, background=False, worker_params=None, timeout=DEFAULT_TIMEOUT_MOVF): """Runs a Powerhose cluster. Options: - **callable**: The Python callable that will be called when the broker receive a job. - **numprocesses**: The number of workers. Defaults to 5. - **frontend**: the ZMQ socket to receive jobs. - **backend**: the ZMQ socket to communicate with workers. - **heartbeat**: the ZMQ socket to receive heartbeat requests/ - **working_dir**: The working directory. Defaults to *"."* - **logfile**: The file to log into. Defaults to stdout. - **debug**: If True, the logs are at the DEBUG level. Defaults to False - **background**: If True, the cluster is run in the background. Defaults to False. - **worker_params**: a dict of params to pass to the worker. Default is None - **timeout** the maximum time allowed before the thread stacks is dumped and the job result not sent back. """ from circus import get_arbiter from circus.stream import StdoutStream, FileStream python = sys.executable if debug: debug = ' --debug' else: debug = '' if worker_params: params = encode_params(worker_params) broker_cmd = [python, '-m', 'powerhose.broker', '--logfile', logfile, debug, '--frontend', frontend, '--backend', backend, '--heartbeat', heartbeat] worker_cmd = [python, '-m', 'powerhose.worker', target, '--logfile', logfile, debug, '--backend', backend, '--heartbeat', heartbeat, '--timeout', str(timeout)] if worker_params: worker_cmd += ['--params', params] if logfile == 'stdout': stream = {'class': StdoutStream} else: stream = {'class': FileStream} watchers = [{'name': 'broker', 'cmd': ' '.join(broker_cmd), 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }, {'name': 'workers', 'cmd': ' '.join(worker_cmd), 'numprocesses': numprocesses, 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream } ] # XXX add more options arbiter = get_arbiter(watchers, background=background) # give a chance to all processes to start # XXX this should be in Circus if background: start = time.clock() while time.clock() - start < 5: statuses = [status == 'active' for status in arbiter.statuses().values()] if all(statuses): break return arbiter
def get_cluster(numprocesses=5, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, working_dir='.', logfile='stdout', debug=False, background=False, worker_params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): """Runs a Loads cluster. Options: - **numprocesses**: The number of workers. Defaults to 5. - **frontend**: the ZMQ socket to receive jobs. - **backend**: the ZMQ socket to communicate with workers. - **register** : the ZMQ socket to register workers - **heartbeat**: the ZMQ socket to receive heartbeat requests - **working_dir**: The working directory. Defaults to *"."* - **logfile**: The file to log into. Defaults to stdout. - **debug**: If True, the logs are at the DEBUG level. Defaults to False - **background**: If True, the cluster is run in the background. Defaults to False. - **worker_params**: a dict of params to pass to the worker. Default is None - **timeout** the maximum time allowed before the thread stacks is dumped and the job result not sent back. - **max_age**: maximum age for a worker in seconds. After that delay, the worker will simply quit. When set to -1, never quits. Defaults to -1. - **max_age_delta**: maximum value in seconds added to max age. The Worker will quit after *max_age + random(0, max_age_delta)* This is done to avoid having all workers quit at the same instant. """ from circus import get_arbiter python = sys.executable if debug: debug = ' --debug' else: debug = '' if worker_params: params = encode_params(worker_params) broker_cmd = [python, '-m', 'loads.transport.broker', '--logfile', logfile, debug, '--frontend', frontend, '--backend', backend, '--heartbeat', heartbeat] worker_cmd = [python, '-m', 'loads.transport.agent', '--logfile', logfile, debug, '--backend', backend, '--heartbeat', heartbeat, '--timeout', str(timeout), '--max-age', str(max_age), '--max-age-delta', str(max_age_delta)] if worker_params: worker_cmd += ['--params', params] if logfile == 'stdout': stream = {'class': 'StdoutStream'} else: stream = {'class': 'FileStream', 'filename': logfile} watchers = [{'name': 'broker', 'cmd': ' '.join(broker_cmd), 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }, {'name': 'workers', 'cmd': ' '.join(worker_cmd), 'numprocesses': numprocesses, 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream } ] # XXX add more options arbiter = get_arbiter(watchers, background=background) # give a chance to all processes to start # XXX this should be in Circus if background: start = time.clock() while time.clock() - start < 5: statuses = [status == 'active' for status in arbiter.statuses().values()] if all(statuses): break return arbiter
def get_cluster(target, numprocesses=5, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, working_dir='.', logfile='stdout', debug=False, background=False, worker_params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): """Runs a Powerhose cluster. Options: - **callable**: The Python callable that will be called when the broker receive a job. - **numprocesses**: The number of workers. Defaults to 5. - **frontend**: the ZMQ socket to receive jobs. - **backend**: the ZMQ socket to communicate with workers. - **register** : the ZMQ socket to register workers - **heartbeat**: the ZMQ socket to receive heartbeat requests - **working_dir**: The working directory. Defaults to *"."* - **logfile**: The file to log into. Defaults to stdout. - **debug**: If True, the logs are at the DEBUG level. Defaults to False - **background**: If True, the cluster is run in the background. Defaults to False. - **worker_params**: a dict of params to pass to the worker. Default is None - **timeout** the maximum time allowed before the thread stacks is dumped and the job result not sent back. - **max_age**: maximum age for a worker in seconds. After that delay, the worker will simply quit. When set to -1, never quits. Defaults to -1. - **max_age_delta**: maximum value in seconds added to max age. The Worker will quit after *max_age + random(0, max_age_delta)* This is done to avoid having all workers quit at the same instant. """ from circus import get_arbiter python = sys.executable if debug: debug = ' --debug' else: debug = '' if worker_params: params = encode_params(worker_params) broker_cmd = [ python, '-m', 'powerhose.broker', '--logfile', logfile, debug, '--frontend', frontend, '--backend', backend, '--heartbeat', heartbeat ] worker_cmd = [ python, '-m', 'powerhose.worker', target, '--logfile', logfile, debug, '--backend', backend, '--heartbeat', heartbeat, '--timeout', str(timeout), '--max-age', str(max_age), '--max-age-delta', str(max_age_delta) ] if worker_params: worker_cmd += ['--params', params] if logfile == 'stdout': stream = {'class': 'StdoutStream'} else: stream = {'class': 'FileStream', 'filename': logfile} watchers = [{ 'name': 'broker', 'cmd': ' '.join(broker_cmd), 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }, { 'name': 'workers', 'cmd': ' '.join(worker_cmd), 'numprocesses': numprocesses, 'working_dir': working_dir, 'executable': python, 'stderr_stream': stream, 'stdout_stream': stream }] # XXX add more options arbiter = get_arbiter(watchers, background=background) # give a chance to all processes to start # XXX this should be in Circus if background: start = time.clock() while time.clock() - start < 5: statuses = [ status == 'active' for status in arbiter.statuses().values() ] if all(statuses): break return arbiter
def __init__(self): self._arbiter = get_arbiter([], background=True) self._arbiter.start() self._client = create_circus_client()
debug=args.debug) with ZeroMQHandler(log_uri, multi=True): logger = Logger("Onitu") ioloop.install() loop = ioloop.IOLoop.instance() arbiter = circus.get_arbiter([ { 'cmd': 'redis-server', 'args': 'redis/redis.conf', 'copy_env': True, 'priority': 1, }, { 'cmd': sys.executable, 'args': ['-m', 'onitu.referee', log_uri], 'copy_env': True, }, ], proc_name="Onitu", loop=loop) for s in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): signal.signal(s, lambda *args, **kwargs: loop.stop()) try: future = arbiter.start() loop.add_future(future, load_drivers) arbiter.start_io_loop()
def main(profile, loggerconfig, foreground, pidfile): """Run an aiida daemon""" import zmq try: zmq_version = [int(part) for part in zmq.__version__.split('.')[:2]] if len(zmq_version) < 2: raise ValueError() except (AttributeError, ValueError): print('Unknown PyZQM version - aborting...') sys.exit(0) if zmq_version[0] < 13 or (zmq_version[0] == 13 and zmq_version[1] < 1): print('circusd needs PyZMQ >= 13.1.0 to run - aborting...') sys.exit(0) loglevel = 'INFO' logoutput = '-' if not foreground: logoutput = 'balrog-{}.log'.format(profile) daemonize() # Create the arbiter profile_config = ProfileConfig(profile) arbiter = get_arbiter( controller=profile_config.get_endpoint(0), pubsub_endpoint=profile_config.get_endpoint(1), stats_endpoint=profile_config.get_endpoint(2), logoutput=logoutput, loglevel=loglevel, debug=False, statsd=True, pidfile='balrog-{}.pid'.format( profile ), # aiida.common.setup.AIIDA_CONFIG_FOLDER + '/daemon/aiida-{}.pid'.format(uuid) watchers=[{ 'name': profile_config.daemon_name, 'cmd': profile_config.cmd_string, 'virtualenv': VIRTUALENV, 'copy_env': True, 'stdout_stream': { 'class': 'FileStream', 'filename': '{}.log'.format(profile_config.daemon_name) }, 'env': { 'PYTHONUNBUFFERED': 'True' } }]) # go ahead and set umask early if it is in the config if arbiter.umask is not None: os.umask(arbiter.umask) pidfile = pidfile or arbiter.pidfile or None if pidfile: pidfile = Pidfile(pidfile) try: pidfile.create(os.getpid()) except RuntimeError as e: print(str(e)) sys.exit(1) # configure the logger loglevel = loglevel or arbiter.loglevel or 'info' logoutput = logoutput or arbiter.logoutput or '-' loggerconfig = loggerconfig or arbiter.loggerconfig or None configure_logger(logger, loglevel, logoutput, loggerconfig) # Main loop restart = True while restart: try: arbiter = arbiter future = arbiter.start() restart = False if check_future_exception_and_log(future) is None: restart = arbiter._restarting except Exception as e: # emergency stop arbiter.loop.run_sync(arbiter._emergency_stop) raise (e) except KeyboardInterrupt: pass finally: arbiter = None if pidfile is not None: pidfile.unlink() sys.exit(0)
def __init__(self, cmd, num_workers=5, timeout=1., check=5., controller='tcp://127.0.0.1:5555', **kw): self.arbiter = get_arbiter(cmd, num_workers, timeout, controller=controller, check_flapping=True, **kw)
from circus import get_arbiter myprogram = {"cmd": "sleep 30", "numprocesses": 4} print('Runnning...') arbiter = get_arbiter([myprogram]) try: arbiter.start() finally: arbiter.stop()
def _start_circus(foreground): """ This will actually launch the circus daemon, either daemonized in the background or in the foreground, printing all logs to stdout. .. note:: this should not be called directly from the commandline! """ from circus import get_arbiter from circus import logger as circus_logger from circus.circusd import daemonize from circus.pidfile import Pidfile from circus.util import check_future_exception_and_log, configure_logger client = DaemonClient() loglevel = client.loglevel logoutput = '-' if not foreground: logoutput = client.circus_log_file arbiter_config = { 'controller': client.get_controller_endpoint(), 'pubsub_endpoint': client.get_pubsub_endpoint(), 'stats_endpoint': client.get_stats_endpoint(), 'logoutput': logoutput, 'loglevel': loglevel, 'debug': False, 'statsd': True, 'pidfile': client.circus_pid_file, 'watchers': [{ 'name': client.daemon_name, 'cmd': client.cmd_string, 'virtualenv': client.virtualenv, 'copy_env': True, 'stdout_stream': { 'class': 'FileStream', 'filename': client.daemon_log_file, }, 'env': get_env_with_venv_bin(), }] } # yapf: disable if not foreground: daemonize() arbiter = get_arbiter(**arbiter_config) pidfile = Pidfile(arbiter.pidfile) try: pidfile.create(os.getpid()) except RuntimeError as exception: click.echo(str(exception)) sys.exit(1) # Configure the logger loggerconfig = None loggerconfig = loggerconfig or arbiter.loggerconfig or None configure_logger(circus_logger, loglevel, logoutput, loggerconfig) # Main loop should_restart = True while should_restart: try: arbiter = arbiter future = arbiter.start() should_restart = False if check_future_exception_and_log(future) is None: should_restart = arbiter._restarting # pylint: disable=protected-access except Exception as exception: # Emergency stop arbiter.loop.run_sync(arbiter._emergency_stop) # pylint: disable=protected-access raise exception except KeyboardInterrupt: pass finally: arbiter = None if pidfile is not None: pidfile.unlink() sys.exit(0)
with ZeroMQHandler(log_uri, multi=True): logger = Logger("Onitu") ioloop.install() loop = ioloop.IOLoop.instance() arbiter = circus.get_arbiter( [ { 'cmd': 'redis-server', 'args': 'redis/redis.conf', 'copy_env': True, 'priority': 1, }, { 'cmd': sys.executable, 'args': ['-m', 'onitu.referee', log_uri], 'copy_env': True, }, ], proc_name="Onitu", loop=loop ) for s in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): signal.signal(s, lambda *args, **kwargs: loop.stop()) try: future = arbiter.start() loop.add_future(future, load_drivers)
def make_arbiter(self): print self.arbiter_config print self.watcher_config return get_arbiter([self.watcher_config], **self.arbiter_config)
from circus import get_arbiter myprogram = { "cmd": "python", "args": "-u dummy_fly.py $(circus.wid)", "numprocesses": 3, } arbiter = get_arbiter([myprogram], debug=True) try: arbiter.start() finally: arbiter.stop()
#!/usr/bin/env python """ This is an addressable worker pool using circus """ from circus import get_arbiter arbiter = get_arbiter([ {'cmd': 'python foo.py'}, {'cmd': 'python bar.py'}, ]) try: arbiter.start() finally: arbiter.stop()
#!/usr/bin/env python3 from circus import get_arbiter sprogram = { "cmd": "python3 stream2db.py", "numprocesses": 1, "copy_env": True, "copy_path": True } arbiter = get_arbiter([sprogram]) try: arbiter.start() finally: arbiter.stop()
myprogram = { "cmd": "python", "args": "-u dummy_fly.py $(circus.wid)", "numprocesses": 3, } from circus import get_arbiter arbiter = get_arbiter([myprogram], debug=True) try: arbiter.start() finally: arbiter.stop()