def __init__(self, endpoint, **kw):
pid = str(os.getpid())
self.endpoint = endpoint.replace('$PID', pid)
pool_kwds = {
'size': 5,
'frontend': self.endpoint,
}
for key in kw:
if key.startswith('pool_'):
pool_kwds[key[5:]] = int(kw[key])
self.pool = Pool(**pool_kwds)
self._metlog = None
try:
res = self.pool.execute(job)
except NoWorkerError:
sys.stdout.write('NO WORKER\n')
else:
assert res == data
sys.stdout.write('OK\n')
sys.stdout.flush()
i += 1
def stop(self):
self.running = False
self.join()
if __name__ == '__main__':
# a pool of 10 workers
client = Pool()
# 10 threads hammering the broker
workers = [Worker(client) for i in range(20)]
for worker in workers:
worker.start()
# just sit there for 100 seconds
try:
time.sleep(3600)
finally:
for worker in workers:
worker.stop()
def __init__(self, endpoint, **kw):
pid = str(os.getpid())
self.endpoint = endpoint.replace('$PID', pid)
self.pool = Pool(int(kw.get('pool_size', 5)), self.endpoint)
self._metlog = None
class PowerHoseRunner(object):
"""Implements a simple powerhose runner.
This class is the one spawning the powerhose master and the workers, if
any need to be created.
You need to instanciate this class with the following parameters::
>>> runner = PowerHoseRunner(endpoint, workers_cmd)
:param endpoint: the zmq endpoint used to communicate between the powerhose
master process and the workers.
This class also provides methods to ease the communication with the
workers. You can directly send information to the workers by using the
methods defined in "methods".
This allows to make calls directly to this object. IOW, it is possible
to call the methods listed in "methods" on the object::
>>> runner.check_signature(**args)
However, all the arguments need to be passed as keyword arguments.
"""
# We implement an interface to be able to retrieve the object with the
# pyramid registry system. This means that this class will only be
# instanciated once, and this instance will be returned each time.
implements(IPowerhoseRunner)
methods = ['derivate_key', 'check_signature', 'check_signature_with_cert',
'is_trusted_issuer']
def __init__(self, endpoint, **kw):
pid = str(os.getpid())
self.endpoint = endpoint.replace('$PID', pid)
self.pool = Pool(int(kw.get('pool_size', 5)), self.endpoint)
self._metlog = None
@property
def metlog(self):
if self._metlog is None:
self._metlog = get_metlog()
return self._metlog
def execute(self, data):
start = time.time()
try:
return self.pool.execute(data)
finally:
duration = time.time() - start
self.metlog.timer_send("token.powerhose", duration)
def __getattr__(self, attr):
"""magic method getter to be able to do direct function calls on this
object.
"""
if attr in self.methods:
return functools.partial(self._execute, attr)
return super(PowerHoseRunner, self).__getattribute__(attr)
def _execute(self, function_id, **data):
"""Send a message to the underlying runner.
This is the low level function, and shouldn't be used directly as-is.
You should use the high level messages to send crypto works to the
workers.
This function takes care of the serialisation / deserialization,
depending the function given.
In the eventuality that the invoked function returns an error, an
exception will be raised.
:param function_id: the name of the function to be invoked.
:param data: the parameters to send to the function.
"""
req_cls, resp_cls = PROTOBUF_CLASSES[function_id]
obj = req_cls()
for key, value in data.items():
setattr(obj, key, value)
# XXX use headers here
data = "::".join((function_id, obj.SerializeToString()))
serialized_resp = self.execute(data)
resp = resp_cls()
try:
resp.ParseFromString(serialized_resp)
except DecodeError:
raise Exception(serialized_resp)
if resp.error_type:
raise ClientCatchedError(resp.error_type, resp.error)
else:
return resp.value
class RSA3248(object):
SIZEINBITS = 3248
def __init__(self):
self.signer = rsa.generate(sizeinbits=self.SIZEINBITS)
def sign(self, msg):
return self.signer.sign(msg)
crypto = RSA3248()
clients = {}
_cl = Pool()
def sign(job):
crypto.sign(job.data)
return 'OK'
@route('/', method='POST')
def index():
data = request.body.read()
crypto.sign(data)
return 'OK'
@route('/phose', method='POST')
class PowerHoseRunner(object):
"""Implements a simple powerhose runner.
This class is the one spawning the powerhose master and the workers, if
any need to be created.
You need to instanciate this class with the following parameters::
runner = PowerHoseRunner(endpoint, workers_cmd)
:param endpoint: the zmq endpoint used to communicate between the powerhose
master process and the workers.
This class also provides methods to ease the communication with the
workers. You can directly send information to the workers by using the
methods defined in "methods".
This allows to make calls directly to this object. IOW, it is possible
to call the methods listed in "methods" on the object::
runner.check_signature(**args)
However, all the arguments need to be passed as keyword arguments.
"""
# We implement an interface to be able to retrieve the object with the
# pyramid registry system. This means that this class will only be
# instanciated once, and this instance will be returned each time.
implements(IPowerhoseRunner)
methods = ['derivate_key', 'check_signature', 'check_signature_with_cert',
'is_trusted_issuer']
def __init__(self, endpoint, **kw):
pid = str(os.getpid())
self.endpoint = endpoint.replace('$PID', pid)
pool_kwds = {
'size': 5,
'frontend': self.endpoint,
}
for key in kw:
if key.startswith('pool_'):
pool_kwds[key[5:]] = int(kw[key])
self.pool = Pool(**pool_kwds)
self._metlog = None
@property
def metlog(self):
if self._metlog is None:
self._metlog = get_metlog()
return self._metlog
def execute(self, data):
start = time.time()
try:
return self.pool.execute(data)
finally:
duration = time.time() - start
self.metlog.timer_send("token.powerhose", duration)
def __getattr__(self, attr):
"""magic method getter to be able to do direct function calls on this
object.
"""
if attr in self.methods:
return functools.partial(self._execute, attr)
return super(PowerHoseRunner, self).__getattribute__(attr)
def _execute(self, function_id, **data):
"""Send a message to the underlying runner.
This is the low level function, and shouldn't be used directly as-is.
You should use the high level messages to send crypto works to the
workers.
This function takes care of the serialisation / deserialization,
depending the function given.
In the eventuality that the invoked function returns an error, an
exception will be raised.
:param function_id: the name of the function to be invoked.
:param data: the parameters to send to the function.
"""
req_cls, resp_cls = PROTOBUF_CLASSES[function_id]
obj = req_cls()
for key, value in data.items():
setattr(obj, key, value)
# XXX use headers here
data = "::".join((function_id, obj.SerializeToString()))
serialized_resp = self.execute(data)
resp = resp_cls()
try:
resp.ParseFromString(serialized_resp)
except DecodeError:
raise Exception(serialized_resp)
if resp.error_type:
raise ClientCatchedError(resp.error_type, resp.error)
else:
return resp.value
