def main(args=sys.argv):
from powerhose.util import set_logger, resolve_name
parser = argparse.ArgumentParser(description='Run a Powerhose cluster.')
parser.add_argument('--frontend',
dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend',
dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for workers.")
parser.add_argument('--heartbeat',
dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('target', help="Fully qualified name of the callable.")
parser.add_argument('--debug',
action='store_true',
default=False,
help="Debug mode")
parser.add_argument('--numprocesses',
dest='numprocesses',
default=5,
help="Number of processes to run.")
parser.add_argument('--logfile',
dest='logfile',
default='stdout',
help="File to log in to .")
args = parser.parse_args()
set_logger(args.debug, 'powerhose', args.logfile)
set_logger(args.debug, 'circus', args.logfile)
sys.path.insert(0, os.getcwd()) # XXX
resolve_name(args.target) # check the callable
cluster = get_cluster(args.target,
args.numprocesses,
frontend=args.frontend,
backend=args.backend,
heartbeat=args.heartbeat,
logfile=args.logfile,
debug=args.debug)
try:
cluster.start()
except KeyboardInterrupt:
pass
finally:
cluster.stop()
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket to the broker.")
parser.add_argument('target', help="Fully qualified name of the callable.")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--params', dest='params', default=None,
help='The parameters to be used in the worker.')
parser.add_argument('--timeout', dest='timeout', type=float,
default=DEFAULT_TIMEOUT_MOVF,
help=('The maximum time allowed before the thread '
'stacks is dump and the job result not sent '
'back.'))
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
sys.path.insert(0, os.getcwd()) # XXX
target = resolve_name(args.target)
if args.params is None:
params = {}
else:
params = decode_params(args.params)
logger.info('Worker registers at %s' % args.backend)
logger.info('The heartbeat socket is at %r' % args.heartbeat)
worker = Worker(target, backend=args.backend, heartbeat=args.heartbeat,
params=params, timeout=args.timeout)
try:
worker.start()
except KeyboardInterrupt:
return 1
finally:
worker.stop()
return 0
def main(args=sys.argv):
from powerhose.util import set_logger, resolve_name
parser = argparse.ArgumentParser(description='Run a Powerhose cluster.')
parser.add_argument('--frontend', dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for workers.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('target', help="Fully qualified name of the callable.")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--numprocesses', dest='numprocesses', default=5,
help="Number of processes to run.")
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to .")
args = parser.parse_args()
set_logger(args.debug, 'powerhose', args.logfile)
set_logger(args.debug, 'circus', args.logfile)
sys.path.insert(0, os.getcwd()) # XXX
resolve_name(args.target) # check the callable
cluster = get_cluster(args.target, args.numprocesses,
frontend=args.frontend, backend=args.backend,
heartbeat=args.heartbeat, logfile=args.logfile,
debug=args.debug)
try:
cluster.start()
except KeyboardInterrupt:
pass
finally:
cluster.stop()
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Powerhose broker.')
parser.add_argument('--frontend', dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for workers.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to .")
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
logger.info('Starting the broker')
broker = Broker(frontend=args.frontend, backend=args.backend,
heartbeat=args.heartbeat)
logger.info('Listening to incoming jobs at %r' % args.frontend)
logger.info('Workers may register at %r' % args.backend)
logger.info('The heartbeat socket is at %r' % args.heartbeat)
try:
broker.start()
except KeyboardInterrupt:
pass
finally:
broker.stop()
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Powerhose broker.')
parser.add_argument('--frontend', dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for workers.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--check', action='store_true', default=False,
help=("Use this option to check if there's a running "
" broker. Returns the PID if a broker is up."))
parser.add_argument('--purge-ghosts', action='store_true', default=False,
help="Use this option to purge ghost brokers.")
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to .")
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
if args.purge_ghosts:
broker_pids, ghosts = kill_ghost_brokers(args.frontend)
if broker_pids is None:
logger.info('No running broker.')
else:
logger.info('The active broker runs at PID: %s' % broker_pids)
if len(ghosts) == 0:
logger.info('No ghosts where killed.')
else:
logger.info('Ghost(s) killed: %s' \
% ', '.join([str(g) for g in ghosts]))
return 0
if args.check:
pid = verify_broker(args.frontend)
if pid is None:
logger.info('There seem to be no broker on this endpoint')
else:
logger.info('A broker is running. PID: %s' % pid)
return 0
logger.info('Starting the broker')
try:
broker = Broker(frontend=args.frontend, backend=args.backend,
heartbeat=args.heartbeat)
except DuplicateBrokerError, e:
logger.info('There is already a broker running on PID %s' % e)
logger.info('Exiting')
return 1
import os
import sys
import binascii
import random
import time
from powerhose.client import Client
from powerhose.job import Job
from powerhose.util import set_logger
set_logger(True)
algs = ('ECDSA256', 'Ed25519', 'RSA2048', 'RSA3248')
client = Client()
def run():
data = binascii.b2a_hex(os.urandom(256))[:256]
job = Job(data + '--' + random.choice(algs))
return client.execute(job)
print 'Running 100 crypto'
start = time.time()
for i in range(100):
res = run()
sys.stdout.write('.')
sys.stdout.flush()
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket to the broker.")
parser.add_argument('target', help="Fully qualified name of the callable.")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--params', dest='params', default=None,
help='The parameters to be used in the worker.')
parser.add_argument('--timeout', dest='timeout', type=float,
default=DEFAULT_TIMEOUT_MOVF,
help=('The maximum time allowed before the thread '
'stacks is dump and the job result not sent '
'back.'))
parser.add_argument('--max-age', dest='max_age', type=float,
default=DEFAULT_MAX_AGE,
help=('The maximum age for a worker in seconds. '
'After that delay, the worker will simply quit. '
'When set to -1, never quits.'))
parser.add_argument('--max-age-delta', dest='max_age_delta', type=int,
default=DEFAULT_MAX_AGE_DELTA,
help='The maximum value in seconds added to max_age')
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
sys.path.insert(0, os.getcwd()) # XXX
target = resolve_name(args.target)
if args.params is None:
params = {}
else:
params = decode_params(args.params)
logger.info('Worker registers at %s' % args.backend)
logger.info('The heartbeat socket is at %r' % args.heartbeat)
worker = Worker(target, backend=args.backend, heartbeat=args.heartbeat,
params=params, timeout=args.timeout, max_age=args.max_age,
max_age_delta=args.max_age_delta)
try:
worker.start()
except KeyboardInterrupt:
return 1
finally:
worker.stop()
return 0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import time
import sys
from powerhose.util import logger, set_logger
set_logger(True, logfile='stdout')
def _p(msg):
sys.stdout.write(msg + '\n')
logger.debug(msg)
sys.stdout.flush()
def fail(job):
_p('Starting powerhose.tests.jobs.fail')
try:
raise ValueError(job.data)
finally:
_p('Ending powerhose.tests.jobs.fail')
def timeout(job):
_p('Starting powerhose.tests.jobs.timeout')
time.sleep(2.)
try:
return job.data
finally:
_p('Ending powerhose.tests.jobs.timeout')
from wsgiref.simple_server import make_server
import json
import time
from powerhose.client import Client
from powerhose.job import Job
from powerhose.util import set_logger
set_logger(True)
client = Client()
def application(environ, start_response):
status = '200 OK'
headers = [('Content-type', 'text/html')]
start_response(status, headers)
data = {}
for key, value in environ.items():
if key.startswith('wsgi.') or key.startswith('gunicorn.'):
continue
data[key] = value
job = Job(json.dumps(data))
start = time.time()
try:
return client.execute(job)
finally:
print('Time : %.4f\n' % (time.time() - start))
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Powerhose broker.')
parser.add_argument('--frontend', dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for workers.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--register', dest='register',
default=DEFAULT_REG,
help="ZMQ socket for the registration.")
parser.add_argument('--io-threads', type=int,
default=DEFAULT_IOTHREADS,
help="Number of I/O threads")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--check', action='store_true', default=False,
help=("Use this option to check if there's a running "
" broker. Returns the PID if a broker is up."))
parser.add_argument('--purge-ghosts', action='store_true', default=False,
help="Use this option to purge ghost brokers.")
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to .")
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
if args.purge_ghosts:
broker_pids, ghosts = kill_ghost_brokers(args.frontend)
if broker_pids is None:
logger.info('No running broker.')
else:
logger.info('The active broker runs at PID: %s' % broker_pids)
if len(ghosts) == 0:
logger.info('No ghosts where killed.')
else:
logger.info('Ghost(s) killed: %s' \
% ', '.join([str(g) for g in ghosts]))
return 0
if args.check:
pid = verify_broker(args.frontend)
if pid is None:
logger.info('There seem to be no broker on this endpoint')
else:
logger.info('A broker is running. PID: %s' % pid)
return 0
logger.info('Starting the broker')
try:
broker = Broker(frontend=args.frontend, backend=args.backend,
heartbeat=args.heartbeat, register=args.register,
io_threads=args.io_threads)
except DuplicateBrokerError, e:
logger.info('There is already a broker running on PID %s' % e)
logger.info('Exiting')
return 1
def main(args=sys.argv):
parser = argparse.ArgumentParser(description="Run some watchers.")
parser.add_argument("--backend", dest="backend", default=DEFAULT_BACKEND, help="ZMQ socket to the broker.")
parser.add_argument("--register", dest="register", default=DEFAULT_REG, help="ZMQ socket for the registration.")
parser.add_argument("target", help="Fully qualified name of the callable.")
parser.add_argument("--debug", action="store_true", default=False, help="Debug mode")
parser.add_argument("--logfile", dest="logfile", default="stdout", help="File to log in to.")
parser.add_argument(
"--heartbeat", dest="heartbeat", default=DEFAULT_HEARTBEAT, help="ZMQ socket for the heartbeat."
)
parser.add_argument("--params", dest="params", default=None, help="The parameters to be used in the worker.")
parser.add_argument(
"--timeout",
dest="timeout",
type=float,
default=DEFAULT_TIMEOUT_MOVF,
help=("The maximum time allowed before the thread " "stacks is dump and the job result not sent " "back."),
)
parser.add_argument(
"--max-age",
dest="max_age",
type=float,
default=DEFAULT_MAX_AGE,
help=(
"The maximum age for a worker in seconds. "
"After that delay, the worker will simply quit. "
"When set to -1, never quits."
),
)
parser.add_argument(
"--max-age-delta",
dest="max_age_delta",
type=int,
default=DEFAULT_MAX_AGE_DELTA,
help="The maximum value in seconds added to max_age",
)
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
sys.path.insert(0, os.getcwd()) # XXX
target = resolve_name(args.target)
if args.params is None:
params = {}
else:
params = decode_params(args.params)
logger.info("Worker registers at %s" % args.backend)
logger.info("The heartbeat socket is at %r" % args.heartbeat)
worker = Worker(
target,
backend=args.backend,
heartbeat=args.heartbeat,
register=args.register,
params=params,
timeout=args.timeout,
max_age=args.max_age,
max_age_delta=args.max_age_delta,
)
try:
worker.start()
except KeyboardInterrupt:
return 1
finally:
worker.stop()
return 0
