def __process_start(self, msg): if msg.payload is None: return Message(name='malformed', success=False) path = msg.payload try: addr = self.__pool.launch(path) return Message(name='return', payload=addr) except ProcessLoadError as ple: return Message(name='failure', success=False, payload=ple.msg)
def output(self, output): """Returns data to the client :param output: the data to send back """ # create the return message reply = Message(name='return', payload=output, success=True) # pack & send off self.__socket.send(reply.pack(defs=self.cruxfile['outputs'])) self.__dirty_socket = False self.__log('returned output')
def fail(self, msg=None): """Fail the current operation :param msg: An optional message to tell the client what's up """ # create the return message reply = Message(name='return', payload=msg, success=False) # pack & send off self.__socket.send(reply.pack()) self.__dirty_socket = False self.__log('returned error message')
def __init__(self, script=None): super().__init__() self.__log = Logger(logging=True, name='dispatch') # set up zmq stuff self.__context = zmq.Context() self.__socket = self.__context.socket(zmq.REQ) self.current_msg = Message() self.__set_prompt() if script is not None: with open(script, 'r') as cmds: self.load(cmds.read().splitlines())
async def send_to_component(self, req): """Send a query to a component HTTP method: POST Query params: address :param req: request from webserver """ if not 'address' in req.query: return self.create_error('"address" is a required parameter!') msg = await req.json() print(msg) if not 'name' in msg: return self.create_error('name not in message!') try: component = self.resolve_component(req.query['address']) except RequestTimeoutException as rte: return self.create_error('request to {} timed out!'.format(req.query['address'])) self.log('component resolved!') # craft the request request = Message(name=msg['name']) if 'payload' in msg: request.payload = msg['payload'] # make the request try: response = component.request(request, timeout=self.REQ_TIMEOUT) except RequestTimeoutException as rte: return self.create_error('request to {} timed out!'.format(req.query['address'])) # prepare the JSON response resp = { 'name': response.name, 'success': response.success, 'payload': response.payload } # respond return web.json_response({ 'response': resp, 'success': True })
def __init__(self, address, socket=None, context=None, timeout=None): """Initialize the component :param address: the address of the component :param socket: the socket to use for communication, if we're sharing (one will be created if not provided) :param timeout: timeout for resolving the component :param context: the context to use to create a socket (if no socket provided), one will be created if not used """ # logging? self.__log = Logger(logging=True) # set up the ZMQ stuff if context is None: if socket is None: self.__log.warn('creating new zmq context for component! something is probably wrong') self.__context = zmq.Context() else: self.__context = context # if we haven't got a socket yet, get one self.__socket = ManagedSocket(self.__context, zmq.REQ) # connect the socket self.__log('connecting to {}'.format(address)) self.__socket.connect(address) # get the cruxfile self.cruxfile = self.request(Message(name='get_cruxfile'), timeout=timeout).payload
def wait(self): """Waits for a client to ask something :returns: a triple (run inputs, run config, done status) """ while True: if self.__dirty_socket: raise NoResultError() msg = Message(data=self.__socket.recv()) reply = Message(name='ack') self.__log('received {}...'.format(msg.name)) # route our reply if msg.name == 'execute': # first check if the request is valid if msg.payload is not None and 'parameters' in msg.payload and 'inputs' in msg.payload: # this is an execution, pass control back to the main loop (but needing closure) self.__log('passing execution back...') self.__dirty_socket = True return (packing.unpack_io_object( msg.payload['inputs'], defs=self.cruxfile['inputs']), self.__defaultify(msg.payload['parameters']), False) else: reply.name = 'malformed' reply.success = False self.__log.error('received malformed execution request') elif msg.name == 'get_cruxfile': reply.payload = self.cruxfile elif msg.name == 'shutdown': break else: # method called has not yet been implemented/is unknown reply.name = 'nyi' reply.success = False # skipped if there was a shutdown or execute instruction self.__socket.send(reply.pack()) self.__dirty_socket = False # if we've broken out this side of the loop, assume we're shutting down self.__log('shutting down loop...') self.__socket.send(reply.pack()) self.__dirty_socket = False return (None, None, True)
def listen(self): self.__log.info('daemon listening on {}'.format(self.__apisock_addr)) # loop until we stop while not self.__should_stop: message = Message(data=self.__apisock.recv()) try: reply = self.__route(message) except Exception as e: reply = Message(name='failure', payload='internal error', success=False) if self.__debug: self.__log.error(e) self.__apisock.send(reply.pack()) self.__log.warn('stopping daemon!') self.__pool.terminate_all()
def __route(self, msg): if msg.name == 'process_start': return self.__process_start(msg) elif msg.name == 'process_list': return self.__process_list(msg) elif msg.name == 'process_killall': return self.__process_killall(msg) elif msg.name == 'daemon_shutdown': return self.__shutdown() return Message(name='nyi', success=False)
def recv(self, timeout=None): """Receive some data on a socket If a timeout is specified, then the socket will be cycled :param timeout: timeout in milliseconds :raises RequestTimeoutException: on timeout :returns: Message """ if timeout is None: return Message(data=self.__socket.recv()) else: # timeouts get a little hairy, but what we're gonna do is: # 1) register socket with a poller # 2) send request # 3) poll for some amount of time # 4) check poller # 5) either return or trash the socket poll = zmq.Poller() poll.register(self.__socket, zmq.POLLIN) # poll socks = dict(poll.poll(timeout)) if socks.get(self.__socket): ret = Message(data=self.__socket.recv()) poll.unregister(self.__socket) return ret else: # the socket is broken, trash it self.__socket.setsockopen(zmq.LINGER, 0) self.__socket.close() poll.unregister(self.__socket) # create a new socket to replace the old one self.__socket = self.__context.socket(self.__socktype) self.__socket.connect(self.__address) raise RequestTimeoutException( 'request to {} timed out'.format(address))
def do_send(self, _): """Send the currently edited message to the connected server""" if self.__addr is None: self.__log.error('not connected!') return try: packed = self.current_msg.pack() except MessageException as me: self.__log.error(me.msg) else: self.__socket.send(packed) self.last_msg = Message(data=self.__socket.recv()) self.__log('sending...')
def run(self, pipeline): """Run the supplied pipeline This function yields every intermediate compuational step. :param pipeline: a dictionary object representing a pipeline """ # first set up all required components for depname in pipeline['components']: dep = pipeline['components'][depname] # launch the process, and bind a component handle to it addr = self.__process_start(dep['src']) self.__cpool[depname] = Component(addr, context=self.__context) # check that the dependency satisfies the version requirements if not version_check(self.__cpool[depname].cruxfile['version'], dep['version']): raise UnmetDependencyError( 'dependency {}@{} does not match requirement {}'.format( depname, self.__cpool[depname].cruxfile['version'], dep['version'])) # kick off the pipeline inp = {} count = 0 for step in pipeline['pipeline']: # perform a request to the specified component result = self.__cpool[step['component']].request( Message(name='execute', payload={ 'parameters': step['parameters'], 'inputs': inp })) # handle results if not result.success: # log & fail self.__log.error('{}: {}'.format(step['component'], result.payload)) raise BrokenPipelineError(result.payload) else: # generators are magic yield (count, step, result) if 'remap' in step: inp = self.__remap_input(result.payload, step['remap']) else: inp = result.payload count += 1
#! /usr/bin/env python import zmq from crux.backend.pool import ProcessPool from crux.pipeline.component import Component from crux.common.messaging import Message context = zmq.Context() pool = ProcessPool() address = pool.launch('../examples/client/') component = Component(address, context=context) print('doing request') result = component.request( Message(name='execute', payload={ 'inputs': 'foo', 'parameters': {} })) print(result) pool.join_all()
def shutdown(self): return self.__call(Message(name='daemon_shutdown'))
def process_list(self): return self.__call(Message(name='process_list'))
def __process_killall(self, msg): self.__log.info('killing all managed processes...') self.__pool.terminate_all() return Message(name='return')
class CruxREPL(cmd.Cmd): # zmq crap __context = None __socket = None __addr = None # message saving and stuff last_msg = None current_msg = None saved_msgs = {} # logging __log = None def __init__(self, script=None): super().__init__() self.__log = Logger(logging=True, name='dispatch') # set up zmq stuff self.__context = zmq.Context() self.__socket = self.__context.socket(zmq.REQ) self.current_msg = Message() self.__set_prompt() if script is not None: with open(script, 'r') as cmds: self.load(cmds.read().splitlines()) def __set_prompt(self): if self.__addr is None: state = colored('disconnected', 'red') else: state = colored(self.__addr, 'green') self.prompt = '({})> '.format(state) def exit(self): if self.__addr is not None: self.do_disconnect(None) def load(self, cmds): stripped = [] for c in cmds: c = c.strip() if c != '': stripped.append(c) self.__log('preloading {}'.format(stripped)) self.cmdqueue.extend(stripped) def message_show(self, msg): if msg is None: print(colored('no message', 'yellow')) return print('{}: {}'.format( colored('name', 'blue'), json.dumps(msg.name) if msg.name is not None else colored( 'None', 'red'))) print('{}: {}'.format( colored('payload', 'blue'), json.dumps(msg.payload) if msg.payload is not None else colored( 'None', 'red'))) print('{}: {}'.format( colored('success', 'blue'), json.dumps(msg.success) if msg.success is not None else colored( 'None', 'red'))) def do_EOF(self, arg): """End the program""" self.exit() print("") return True def do_exit(self, arg): """End the program""" self.exit() return True def do_connect(self, addr): """Connect to an address""" if self.__addr is not None: self.__log.warn( 'socket was connected to {}, disconnecting...'.format( self.__addr)) self.__socket.disconnect(self.__addr) self.__addr = addr self.__socket.connect(self.__addr) self.__log.info('connected to {}'.format(self.__addr)) self.__set_prompt() def do_disconnect(self, args): """Disconnect""" if self.__addr is not None: self.__socket.disconnect(self.__addr) self.__log.info('disconnected from {}'.format(self.__addr)) else: self.__log.warn('socket was not connected, ignoring...') self.__addr = None self.__set_prompt() def do_cset(self, args): """set the (payload/name/success) currently edited message. cset [field] [value]""" try: field, data = args.split(' ', 1) except: self.__log.error('error parsing arguments') return if not field in ['name', 'payload', 'success']: self.__log.error('invalid field \'{}\''.format(field)) return try: setattr(self.current_msg, field, json.loads(data)) except json.decoder.JSONDecodeError as jde: self.__log.error('invalid data \'{}\''.format(data)) else: self.__log.info('set field {} to {}'.format(field, data)) def do_cshow(self, _): """Show the currently edited message""" self.message_show(self.current_msg) def do_creset(self, _): """Reset the current message""" self.current_msg = Message() def do_lshow(self, _): """Show the last received message""" self.message_show(self.last_msg) def do_lreset(self, _): """Reset the last received message""" self.last_msg = None def do_echo(self, txt): """Echo some text""" print('[{}]: {}'.format(colored('echo ', 'magenta'), txt)) def do_send(self, _): """Send the currently edited message to the connected server""" if self.__addr is None: self.__log.error('not connected!') return try: packed = self.current_msg.pack() except MessageException as me: self.__log.error(me.msg) else: self.__socket.send(packed) self.last_msg = Message(data=self.__socket.recv()) self.__log('sending...') def do_assert(self, args): """Make an assertion that the args are equal. assert [field] [value]""" try: field, data = args.split(' ', 1) data = json.loads(data) except: self.__log.error('error parsing arguments') return # resolve the path def resolve_path(obj, fieldpath): if type(obj) is list: fieldpath[0] = int(fieldpath[0]) if len(fieldpath) == 0: return obj if len(fieldpath) == 1: return obj[fieldpath[0]] else: return resolve_path(obj[fieldpath[0]], fieldpath[1:]) try: field = field.split('.') compare = resolve_path(getattr(self.last_msg, field[0]), field[1:]) except: self.__log.warn( 'assertion warning: field path expansion failed for {}'.format( field)) compare = None if compare == data: self.__log.info('assertion passed: {}'.format(data)) else: self.__log.error('assertion failed!: {} != {}'.format( json.dumps(data), json.dumps(compare)))
def __shutdown(self): self.__should_stop = True return Message(name='return')
def do_creset(self, _): """Reset the current message""" self.current_msg = Message()
def process_start(self, path): return self.__call(Message(name='process_start', payload=path))
def process_killall(self): return self.__call(Message(name='process_killall'))
def __process_list(self, msg): return Message(name='return', payload=self.__pool.get_all_addrs())