def __process_start(self, msg):
if msg.payload is None:
return Message(name='malformed', success=False)
path = msg.payload
try:
addr = self.__pool.launch(path)
return Message(name='return', payload=addr)
except ProcessLoadError as ple:
return Message(name='failure', success=False, payload=ple.msg)
def output(self, output):
"""Returns data to the client
:param output: the data to send back
"""
# create the return message
reply = Message(name='return', payload=output, success=True)
# pack & send off
self.__socket.send(reply.pack(defs=self.cruxfile['outputs']))
self.__dirty_socket = False
self.__log('returned output')
def fail(self, msg=None):
"""Fail the current operation
:param msg: An optional message to tell the client what's up
"""
# create the return message
reply = Message(name='return', payload=msg, success=False)
# pack & send off
self.__socket.send(reply.pack())
self.__dirty_socket = False
self.__log('returned error message')
def __init__(self, script=None):
super().__init__()
self.__log = Logger(logging=True, name='dispatch')
# set up zmq stuff
self.__context = zmq.Context()
self.__socket = self.__context.socket(zmq.REQ)
self.current_msg = Message()
self.__set_prompt()
if script is not None:
with open(script, 'r') as cmds:
self.load(cmds.read().splitlines())
async def send_to_component(self, req):
"""Send a query to a component
HTTP method: POST
Query params: address
:param req: request from webserver
"""
if not 'address' in req.query:
return self.create_error('"address" is a required parameter!')
msg = await req.json()
print(msg)
if not 'name' in msg:
return self.create_error('name not in message!')
try:
component = self.resolve_component(req.query['address'])
except RequestTimeoutException as rte:
return self.create_error('request to {} timed out!'.format(req.query['address']))
self.log('component resolved!')
# craft the request
request = Message(name=msg['name'])
if 'payload' in msg:
request.payload = msg['payload']
# make the request
try:
response = component.request(request, timeout=self.REQ_TIMEOUT)
except RequestTimeoutException as rte:
return self.create_error('request to {} timed out!'.format(req.query['address']))
# prepare the JSON response
resp = {
'name': response.name,
'success': response.success,
'payload': response.payload
}
# respond
return web.json_response({
'response': resp,
'success': True
})
def __init__(self, address, socket=None, context=None, timeout=None):
"""Initialize the component
:param address: the address of the component
:param socket: the socket to use for communication, if we're sharing (one will be created if not provided)
:param timeout: timeout for resolving the component
:param context: the context to use to create a socket (if no socket provided), one will be created if not used
"""
# logging?
self.__log = Logger(logging=True)
# set up the ZMQ stuff
if context is None:
if socket is None:
self.__log.warn('creating new zmq context for component! something is probably wrong')
self.__context = zmq.Context()
else:
self.__context = context
# if we haven't got a socket yet, get one
self.__socket = ManagedSocket(self.__context, zmq.REQ)
# connect the socket
self.__log('connecting to {}'.format(address))
self.__socket.connect(address)
# get the cruxfile
self.cruxfile = self.request(Message(name='get_cruxfile'), timeout=timeout).payload
def wait(self):
"""Waits for a client to ask something
:returns: a triple (run inputs, run config, done status)
"""
while True:
if self.__dirty_socket:
raise NoResultError()
msg = Message(data=self.__socket.recv())
reply = Message(name='ack')
self.__log('received {}...'.format(msg.name))
# route our reply
if msg.name == 'execute':
# first check if the request is valid
if msg.payload is not None and 'parameters' in msg.payload and 'inputs' in msg.payload:
# this is an execution, pass control back to the main loop (but needing closure)
self.__log('passing execution back...')
self.__dirty_socket = True
return (packing.unpack_io_object(
msg.payload['inputs'], defs=self.cruxfile['inputs']),
self.__defaultify(msg.payload['parameters']),
False)
else:
reply.name = 'malformed'
reply.success = False
self.__log.error('received malformed execution request')
elif msg.name == 'get_cruxfile':
reply.payload = self.cruxfile
elif msg.name == 'shutdown':
break
else:
# method called has not yet been implemented/is unknown
reply.name = 'nyi'
reply.success = False
# skipped if there was a shutdown or execute instruction
self.__socket.send(reply.pack())
self.__dirty_socket = False
# if we've broken out this side of the loop, assume we're shutting down
self.__log('shutting down loop...')
self.__socket.send(reply.pack())
self.__dirty_socket = False
return (None, None, True)
def listen(self):
self.__log.info('daemon listening on {}'.format(self.__apisock_addr))
# loop until we stop
while not self.__should_stop:
message = Message(data=self.__apisock.recv())
try:
reply = self.__route(message)
except Exception as e:
reply = Message(name='failure',
payload='internal error',
success=False)
if self.__debug:
self.__log.error(e)
self.__apisock.send(reply.pack())
self.__log.warn('stopping daemon!')
self.__pool.terminate_all()
def __route(self, msg):
if msg.name == 'process_start':
return self.__process_start(msg)
elif msg.name == 'process_list':
return self.__process_list(msg)
elif msg.name == 'process_killall':
return self.__process_killall(msg)
elif msg.name == 'daemon_shutdown':
return self.__shutdown()
return Message(name='nyi', success=False)
def recv(self, timeout=None):
"""Receive some data on a socket
If a timeout is specified, then the socket will be cycled
:param timeout: timeout in milliseconds
:raises RequestTimeoutException: on timeout
:returns: Message
"""
if timeout is None:
return Message(data=self.__socket.recv())
else:
# timeouts get a little hairy, but what we're gonna do is:
# 1) register socket with a poller
# 2) send request
# 3) poll for some amount of time
# 4) check poller
# 5) either return or trash the socket
poll = zmq.Poller()
poll.register(self.__socket, zmq.POLLIN)
# poll
socks = dict(poll.poll(timeout))
if socks.get(self.__socket):
ret = Message(data=self.__socket.recv())
poll.unregister(self.__socket)
return ret
else:
# the socket is broken, trash it
self.__socket.setsockopen(zmq.LINGER, 0)
self.__socket.close()
poll.unregister(self.__socket)
# create a new socket to replace the old one
self.__socket = self.__context.socket(self.__socktype)
self.__socket.connect(self.__address)
raise RequestTimeoutException(
'request to {} timed out'.format(address))
def do_send(self, _):
"""Send the currently edited message to the connected server"""
if self.__addr is None:
self.__log.error('not connected!')
return
try:
packed = self.current_msg.pack()
except MessageException as me:
self.__log.error(me.msg)
else:
self.__socket.send(packed)
self.last_msg = Message(data=self.__socket.recv())
self.__log('sending...')
def run(self, pipeline):
"""Run the supplied pipeline
This function yields every intermediate compuational step.
:param pipeline: a dictionary object representing a pipeline
"""
# first set up all required components
for depname in pipeline['components']:
dep = pipeline['components'][depname]
# launch the process, and bind a component handle to it
addr = self.__process_start(dep['src'])
self.__cpool[depname] = Component(addr, context=self.__context)
# check that the dependency satisfies the version requirements
if not version_check(self.__cpool[depname].cruxfile['version'],
dep['version']):
raise UnmetDependencyError(
'dependency {}@{} does not match requirement {}'.format(
depname, self.__cpool[depname].cruxfile['version'],
dep['version']))
# kick off the pipeline
inp = {}
count = 0
for step in pipeline['pipeline']:
# perform a request to the specified component
result = self.__cpool[step['component']].request(
Message(name='execute',
payload={
'parameters': step['parameters'],
'inputs': inp
}))
# handle results
if not result.success:
# log & fail
self.__log.error('{}: {}'.format(step['component'],
result.payload))
raise BrokenPipelineError(result.payload)
else:
# generators are magic
yield (count, step, result)
if 'remap' in step:
inp = self.__remap_input(result.payload, step['remap'])
else:
inp = result.payload
count += 1
#! /usr/bin/env python
import zmq
from crux.backend.pool import ProcessPool
from crux.pipeline.component import Component
from crux.common.messaging import Message
context = zmq.Context()
pool = ProcessPool()
address = pool.launch('../examples/client/')
component = Component(address, context=context)
print('doing request')
result = component.request(
Message(name='execute', payload={
'inputs': 'foo',
'parameters': {}
}))
print(result)
pool.join_all()
def shutdown(self):
return self.__call(Message(name='daemon_shutdown'))
def process_list(self):
return self.__call(Message(name='process_list'))
def __process_killall(self, msg):
self.__log.info('killing all managed processes...')
self.__pool.terminate_all()
return Message(name='return')
class CruxREPL(cmd.Cmd):
# zmq crap
__context = None
__socket = None
__addr = None
# message saving and stuff
last_msg = None
current_msg = None
saved_msgs = {}
# logging
__log = None
def __init__(self, script=None):
super().__init__()
self.__log = Logger(logging=True, name='dispatch')
# set up zmq stuff
self.__context = zmq.Context()
self.__socket = self.__context.socket(zmq.REQ)
self.current_msg = Message()
self.__set_prompt()
if script is not None:
with open(script, 'r') as cmds:
self.load(cmds.read().splitlines())
def __set_prompt(self):
if self.__addr is None:
state = colored('disconnected', 'red')
else:
state = colored(self.__addr, 'green')
self.prompt = '({})> '.format(state)
def exit(self):
if self.__addr is not None:
self.do_disconnect(None)
def load(self, cmds):
stripped = []
for c in cmds:
c = c.strip()
if c != '':
stripped.append(c)
self.__log('preloading {}'.format(stripped))
self.cmdqueue.extend(stripped)
def message_show(self, msg):
if msg is None:
print(colored('no message', 'yellow'))
return
print('{}: {}'.format(
colored('name', 'blue'),
json.dumps(msg.name) if msg.name is not None else colored(
'None', 'red')))
print('{}: {}'.format(
colored('payload', 'blue'),
json.dumps(msg.payload) if msg.payload is not None else colored(
'None', 'red')))
print('{}: {}'.format(
colored('success', 'blue'),
json.dumps(msg.success) if msg.success is not None else colored(
'None', 'red')))
def do_EOF(self, arg):
"""End the program"""
self.exit()
print("")
return True
def do_exit(self, arg):
"""End the program"""
self.exit()
return True
def do_connect(self, addr):
"""Connect to an address"""
if self.__addr is not None:
self.__log.warn(
'socket was connected to {}, disconnecting...'.format(
self.__addr))
self.__socket.disconnect(self.__addr)
self.__addr = addr
self.__socket.connect(self.__addr)
self.__log.info('connected to {}'.format(self.__addr))
self.__set_prompt()
def do_disconnect(self, args):
"""Disconnect"""
if self.__addr is not None:
self.__socket.disconnect(self.__addr)
self.__log.info('disconnected from {}'.format(self.__addr))
else:
self.__log.warn('socket was not connected, ignoring...')
self.__addr = None
self.__set_prompt()
def do_cset(self, args):
"""set the (payload/name/success) currently edited message. cset [field] [value]"""
try:
field, data = args.split(' ', 1)
except:
self.__log.error('error parsing arguments')
return
if not field in ['name', 'payload', 'success']:
self.__log.error('invalid field \'{}\''.format(field))
return
try:
setattr(self.current_msg, field, json.loads(data))
except json.decoder.JSONDecodeError as jde:
self.__log.error('invalid data \'{}\''.format(data))
else:
self.__log.info('set field {} to {}'.format(field, data))
def do_cshow(self, _):
"""Show the currently edited message"""
self.message_show(self.current_msg)
def do_creset(self, _):
"""Reset the current message"""
self.current_msg = Message()
def do_lshow(self, _):
"""Show the last received message"""
self.message_show(self.last_msg)
def do_lreset(self, _):
"""Reset the last received message"""
self.last_msg = None
def do_echo(self, txt):
"""Echo some text"""
print('[{}]: {}'.format(colored('echo ', 'magenta'), txt))
def do_send(self, _):
"""Send the currently edited message to the connected server"""
if self.__addr is None:
self.__log.error('not connected!')
return
try:
packed = self.current_msg.pack()
except MessageException as me:
self.__log.error(me.msg)
else:
self.__socket.send(packed)
self.last_msg = Message(data=self.__socket.recv())
self.__log('sending...')
def do_assert(self, args):
"""Make an assertion that the args are equal. assert [field] [value]"""
try:
field, data = args.split(' ', 1)
data = json.loads(data)
except:
self.__log.error('error parsing arguments')
return
# resolve the path
def resolve_path(obj, fieldpath):
if type(obj) is list:
fieldpath[0] = int(fieldpath[0])
if len(fieldpath) == 0:
return obj
if len(fieldpath) == 1:
return obj[fieldpath[0]]
else:
return resolve_path(obj[fieldpath[0]], fieldpath[1:])
try:
field = field.split('.')
compare = resolve_path(getattr(self.last_msg, field[0]), field[1:])
except:
self.__log.warn(
'assertion warning: field path expansion failed for {}'.format(
field))
compare = None
if compare == data:
self.__log.info('assertion passed: {}'.format(data))
else:
self.__log.error('assertion failed!: {} != {}'.format(
json.dumps(data), json.dumps(compare)))
def __shutdown(self):
self.__should_stop = True
return Message(name='return')
def do_creset(self, _):
"""Reset the current message"""
self.current_msg = Message()
def process_start(self, path):
return self.__call(Message(name='process_start', payload=path))
def process_killall(self):
return self.__call(Message(name='process_killall'))
def __process_list(self, msg):
return Message(name='return', payload=self.__pool.get_all_addrs())