def setup(self, message): # Spawn the remote hive self.remote_hive_id = base64_uuid4() self.send_queue = Queue() self.receive_queue = Queue() self.multiproces_hive_proc = Process( target=spawn_multiprocess_hive, args=( self.remote_hive_id, # The opposite of our send and receive queue! self.send_queue, self.receive_queue)) self.multiproces_hive_proc.start() # Declare ourselves the ambassador for this hive self.send_message( to=join_id("hive", self.hive.hive_id), directive="register_ambassador", body={ "hive_id": self.remote_hive_id}) # Set up the message-checking loop self.send_message( to=self.id, directive="check_message_loop") # Tell the child hive to connect back to us yield self.wait_on_message( to=join_id("hive", self.remote_hive_id), directive="connect_back", body={"parent_hive_id": self.hive.hive_id})
def setup(self, message): # Spawn the remote hive self.remote_hive_id = base64_uuid4() self.send_queue = Queue() self.receive_queue = Queue() self.multiproces_hive_proc = Process( target=spawn_multiprocess_hive, args=( self.remote_hive_id, # The opposite of our send and receive queue! self.send_queue, self.receive_queue)) self.multiproces_hive_proc.start() # Declare ourselves the ambassador for this hive self.send_message(to=join_id("hive", self.hive.hive_id), directive="register_ambassador", body={"hive_id": self.remote_hive_id}) # Set up the message-checking loop self.send_message(to=self.id, directive="check_message_loop") # Tell the child hive to connect back to us yield self.wait_on_message(to=join_id("hive", self.remote_hive_id), directive="connect_back", body={"parent_hive_id": self.hive.hive_id})
def oversee_experiments(self, message): yield self.wait_on_message( to=self.id, directive="setup_worker_processes") num_experiments = message.body['num_experiments'] num_steps = message.body['num_steps'] slacker_time = message.body['slacker_time'] if message.body.get('success_tracker'): self.success_tracker = message.body['success_tracker'] print("Starting %s experiments with %s steps each" % ( num_experiments, num_steps)) # A lazy hack to avoid race conditions run_experiments = [] allocation = worker_allocation( range(num_experiments), self.worker_hives) for i, hive_id in allocation: response = yield self.wait_on_message( to=join_id("hive", hive_id), directive="create_actor", body={ "class": "xudd.demos.lotsamessages:Professor"}) professor = response.body['actor_id'] response = yield self.wait_on_message( to=join_id("hive", hive_id), directive="create_actor", body={ "class": "xudd.demos.lotsamessages:Assistant"}) assistant = response.body['actor_id'] self.experiments_in_progress.add(professor) run_experiments.append((professor, assistant)) # We do this on a separate loop to avoid the race conditions where some # professors finish up before others even start for professor, assistant in run_experiments: self.hive.send_message( to=professor, directive="run_experiments", body={ "assistant_id": assistant, "numtimes": num_steps, "slacker_time": slacker_time})
def __init__(self, hive_id=None, loop=None): # id of the hive self.hive_id = hive_id or self.gen_actor_id() hive_proxy = self.gen_proxy() super(Hive, self).__init__( id= join_id(self.hive_id, self.hive_id), hive=hive_proxy) hive_proxy.associate_with_actor(self) # Which actors this hive is managing self._actor_registry = {} # Note: can we just trust the user to set the right policy? self.loop = loop or asyncio.get_event_loop() # Objects related to generating unique ids for messages self.message_uuid = base64_uuid4() self.message_counter = count() # Ambassador registry (for inter-hive-communication) self._ambassadors = {} # Extend message routing self.message_routing.update( {"register_ambassador": self.register_ambassador, "unregister_ambassador": self.unregister_ambassador, "create_actor": self.create_actor_handler}) # Register ourselves on... ourselves ;) self.register_actor(self)
def listen(self, message): body = message.body port = body.get('port', 8000) host = body.get('host', '127.0.0.1') self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setblocking(0) # XXX: Don't know if this helps much self.socket.bind((host, port)) self.socket.listen(5) # Max 5 connections in queue while True: readable, writable, errored = select.select( [self.socket], [], [], .0000001) # XXX: This will surely make it fast! (?) if readable: _log.info('Got new request ({0} in local index)'.format( len(self.requests))) req = self.socket.accept() # Use the message id as the internal id for the request message_id = self.send_message(to=join_id( 'http', self.hive.hive_id), directive='handle_request', body={'request': req}) _log.debug('Sent request to worker') self.requests.update({message_id: req}) yield self.wait_on_self()
def __init__(self, hive_id=None, loop=None): # id of the hive self.hive_id = hive_id or self.gen_actor_id() hive_proxy = self.gen_proxy() super(Hive, self).__init__( id=join_id("hive", self.hive_id), hive=hive_proxy) hive_proxy.associate_with_actor(self) # Which actors this hive is managing self._actor_registry = {} # Note: can we just trust the user to set the right policy? self.loop = loop or asyncio.get_event_loop() # Objects related to generating unique ids for messages self.message_uuid = base64_uuid4() self.message_counter = count() # Ambassador registry (for inter-hive-communication) self._ambassadors = {} # Extend message routing self.message_routing.update( {"register_ambassador": self.register_ambassador, "unregister_ambassador": self.unregister_ambassador, "create_actor": self.create_actor_handler}) # Register ourselves on... ourselves ;) self.register_actor(self)
def oversee_experiments(self, message): yield self.wait_on_message(to=self.id, directive="setup_worker_processes") num_experiments = message.body['num_experiments'] num_steps = message.body['num_steps'] slacker_time = message.body['slacker_time'] if message.body.get('success_tracker'): self.success_tracker = message.body['success_tracker'] print("Starting %s experiments with %s steps each" % (num_experiments, num_steps)) # A lazy hack to avoid race conditions run_experiments = [] allocation = worker_allocation(range(num_experiments), self.worker_hives) for i, hive_id in allocation: response = yield self.wait_on_message( to=join_id("hive", hive_id), directive="create_actor", body={"class": "xudd.demos.lotsamessages:Professor"}) professor = response.body['actor_id'] response = yield self.wait_on_message( to=join_id("hive", hive_id), directive="create_actor", body={"class": "xudd.demos.lotsamessages:Assistant"}) assistant = response.body['actor_id'] self.experiments_in_progress.add(professor) run_experiments.append((professor, assistant)) # We do this on a separate loop to avoid the race conditions where some # professors finish up before others even start for professor, assistant in run_experiments: self.hive.send_message(to=professor, directive="run_experiments", body={ "assistant_id": assistant, "numtimes": num_steps, "slacker_time": slacker_time })
def create_actor(self, actor_class, *args, **kwargs): hive_proxy = self.gen_proxy() actor_id = join_id( kwargs.pop("id", None) or self.gen_actor_id(), self.hive_id) actor = actor_class(hive_proxy, actor_id, *args, **kwargs) hive_proxy.associate_with_actor(actor) self.register_actor(actor) return actor_id
def nerd_out_to_hive(self, message): response = yield self.wait_on_message(to=join_id( "hive", self.hive.hive_id), directive="be_fanboyed") assert response.body['is_hive'] == True assert response.body['vocal_tone'] == 'confident' print("Omg! The hive wrote back to me! They're SO CONFIDENT... <3") print("And look! They included a message:") print(' "%s"' % response.body["words_of_wisdom"]) self.hive.send_shutdown()
def nerd_out_to_hive(self, message): response = yield self.wait_on_message( to=join_id("hive", self.hive.hive_id), directive="be_fanboyed") assert response.body['is_hive'] == True assert response.body['vocal_tone'] == 'confident' print("Omg! The hive wrote back to me! They're SO CONFIDENT... <3") print("And look! They included a message:") print(' "%s"' % response.body["words_of_wisdom"]) self.hive.send_shutdown()
def create_actor(self, actor_class, *args, **kwargs): hive_proxy = self.gen_proxy() actor_id = join_id( kwargs.pop("id", None) or self.gen_actor_id(), self.hive_id) actor = actor_class( hive_proxy, actor_id, *args, **kwargs) hive_proxy.associate_with_actor(actor) self.register_actor(actor) return actor_id
def experiment_is_done(self, message): self.experiments_in_progress.remove(message.from_id) print("%s experiment is done" % message.from_id) if len(self.experiments_in_progress) == 0: print( "Last experiment message (%s) received from (%s), shutting down" % (message.id, message.from_id)) if self.success_tracker is not None: self.success_tracker.success = True # Shut down all child hives too if self.num_worker_processes > 0: for worker_hive in self.worker_hives: yield self.wait_on_message(to=join_id("hive", worker_hive), directive="remote_shutdown") self.hive.send_shutdown()
def experiment_is_done(self, message): self.experiments_in_progress.remove(message.from_id) print("%s experiment is done" % message.from_id) if len(self.experiments_in_progress) == 0: print( "Last experiment message (%s) received from (%s), shutting down" % ( message.id, message.from_id)) if self.success_tracker is not None: self.success_tracker.success = True # Shut down all child hives too if self.num_worker_processes > 0: for worker_hive in self.worker_hives: yield self.wait_on_message( to=join_id("hive", worker_hive), directive="remote_shutdown") self.hive.send_shutdown()
def listen(self, message): body = message.body port = body.get('port', 8000) host = body.get('host', '127.0.0.1') self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setblocking(0) # XXX: Don't know if this helps much self.socket.bind((host, port)) self.socket.listen(5) # Max 5 connections in queue while True: readable, writable, errored = select.select( [self.socket], [], [], .0000001) # XXX: This will surely make it fast! (?) if readable: _log.info('Got new request ({0} in local index)'.format(len(self.requests))) req = self.socket.accept() # Use the message id as the internal id for the request message_id = self.send_message( to=join_id('http', self.hive.hive_id), directive='handle_request', body={ 'request': req } ) _log.debug('Sent request to worker') self.requests.update({ message_id: req }) yield self.wait_on_self()
def handle_request(self, message): ''' Much of the code for parsing HTTP is inspired by the tornado framweork <http://tornadoweb.org>. ''' sock, bind = message.body['request'] while True: r, w, e = select.select([sock], [], [], .0001) if not r: yield self.wait_on_self() else: first_data = sock.recv(8192) _log.debug('first_data: {0}'.format(first_data)) # XXX: Sometimes first_data is zero-length when running # `ab -n 10000 -c 500 URI` against the server, this block # catches those cases, but I'm still not sure why they occur. try: request_line, rest = first_data.split('\r\n', 1) method, uri, version = request_line.split(' ') http_headers, rest = rest.split('\r\n\r\n', 1) headers = httputil.HTTPHeaders.parse(http_headers) _log.info('headers: {0}'.format(headers)) remote_ip = sock.getpeername()[0] content_length = headers.get('Content-Length') if content_length: content_length = int(content_length) if content_length > MAX_REQUEST_SIZE: raise Exception('Content-Length too long') if headers.get('Expect') == '100-continue': sock.sendall('HTTP/1.1 100 (Continue)\r\n\r\n') additional_data = b'' while True: r, w, e = select.select([sock], [], [], .0001) if not r: yield self.wait_on_self() additional_data += sock.recv(content_length) if len(additional_data) == content_length: break _log.debug('additional_data: {0}'.format( additional_data)) rest += additional_data body = rest _log.debug('body: {0}'.format(body)) option_names = ('method', 'uri', 'version', 'headers', 'remote_ip', 'content_length') options = dict( method=method, uri=uri, version=version, headers=headers, remote_ip=remote_ip, content_length=content_length, server_name=bind[0], port=bind[1]) _log.debug('options: {0}'.format(options)) _log.info('{method} {uri} ({content_length})'.format( **options)) except Exception as exc: _log.error('Failed to parse request: {0}\n---\n{0}'.format( traceback.format_exc(), first_data )) message.reply( directive='respond', body={ 'response': 'HTTP/1.1 400 Invalid Request\r\n' 'Connection: close' }) break # Don't try to parse the request any further as # we've already replied with a 400 parsed = yield self.wait_on_message( to=self.id, directive='handle_request_body', body={ 'options': options, 'body': body } ) response = yield self.wait_on_message( to=join_id('wsgi', self.hive.hive_id), directive='handle_request', body={ 'body': body, 'options': options, 'arguments': parsed.body['arguments'], 'files': parsed.body['files'] }) message.reply( directive='respond', body={ 'response': response.body.get('response') }) break
def begin_mission(self, message): self.room = message.body['starting_room'] overseer_id = join_id("overseer", self.hive.hive_id) # Walk through all rooms, clearing out infected droids while True: self.hive.send_message( to=overseer_id, directive="transmission", body={"message": "Entering room %s..." % self.room}) # Find all the droids in this room and exterminate the # infected ones. response = yield self.wait_on_message(to=self.room, directive="list_droids") for droid_id in response.body["droid_ids"]: response = yield self.wait_on_message( to=droid_id, directive="infection_expose") # If the droid is clean, let the overseer know and move on. if not response.body["is_infected"]: transmission = ("%s is clean... moving on." % droid_id) self.hive.send_message(to=overseer_id, directive="transmission", body={"message": transmission}) continue # Let the overseer know we found an infected droid # and are engaging transmission = ("%s found to be infected... taking out" % droid_id) self.hive.send_message(to=overseer_id, directive="transmission", body={"message": transmission}) # Keep firing till it's dead. infected_droid_alive = True while infected_droid_alive: response = yield self.wait_on_message(to=droid_id, directive="get_shot") # Relay the droid status droid_status = self.__droid_status_format(response) self.hive.send_message(to=overseer_id, directive="transmission", body={"message": droid_status}) infected_droid_alive = response.body["alive"] # switch to next room, if there is one response = yield self.wait_on_message(to=self.room, directive="get_next_room") next_room = response.body["id"] if next_room: self.room = next_room else: # We're done scanning rooms finally break # Good job everyone! Shut down the operation. yield self.wait_on_message(to=overseer_id, directive="transmission", body={"message": "Mission accomplished."}) self.hive.send_shutdown()
def begin_mission(self, message): self.room = message.body['starting_room'] overseer_id = join_id("overseer", self.hive.hive_id) # Walk through all rooms, clearing out infected droids while True: self.hive.send_message( to=overseer_id, directive="transmission", body={ "message": "Entering room %s..." % self.room}) # Find all the droids in this room and exterminate the # infected ones. response = yield self.wait_on_message( to=self.room, directive="list_droids") for droid_id in response.body["droid_ids"]: response = yield self.wait_on_message( to=droid_id, directive="infection_expose") # If the droid is clean, let the overseer know and move on. if not response.body["is_infected"]: transmission = ( "%s is clean... moving on." % droid_id) self.hive.send_message( to=overseer_id, directive="transmission", body={ "message": transmission}) continue # Let the overseer know we found an infected droid # and are engaging transmission = ( "%s found to be infected... taking out" % droid_id) self.hive.send_message( to=overseer_id, directive="transmission", body={ "message": transmission}) # Keep firing till it's dead. infected_droid_alive = True while infected_droid_alive: response = yield self.wait_on_message( to=droid_id, directive="get_shot") # Relay the droid status droid_status = self.__droid_status_format(response) self.hive.send_message( to=overseer_id, directive="transmission", body={ "message": droid_status}) infected_droid_alive = response.body["alive"] # switch to next room, if there is one response = yield self.wait_on_message( to=self.room, directive="get_next_room") next_room = response.body["id"] if next_room: self.room = next_room else: # We're done scanning rooms finally break # Good job everyone! Shut down the operation. yield self.wait_on_message( to=overseer_id, directive="transmission", body={ "message": "Mission accomplished."}) self.hive.send_shutdown()
def handle_request(self, message): ''' Much of the code for parsing HTTP is inspired by the tornado framweork <http://tornadoweb.org>. ''' sock, bind = message.body['request'] while True: r, w, e = select.select([sock], [], [], .0001) if not r: yield self.wait_on_self() else: first_data = sock.recv(8192) _log.debug('first_data: {0}'.format(first_data)) # XXX: Sometimes first_data is zero-length when running # `ab -n 10000 -c 500 URI` against the server, this block # catches those cases, but I'm still not sure why they occur. try: request_line, rest = first_data.split('\r\n', 1) method, uri, version = request_line.split(' ') http_headers, rest = rest.split('\r\n\r\n', 1) headers = httputil.HTTPHeaders.parse(http_headers) _log.info('headers: {0}'.format(headers)) remote_ip = sock.getpeername()[0] content_length = headers.get('Content-Length') if content_length: content_length = int(content_length) if content_length > MAX_REQUEST_SIZE: raise Exception('Content-Length too long') if headers.get('Expect') == '100-continue': sock.sendall('HTTP/1.1 100 (Continue)\r\n\r\n') additional_data = b'' while True: r, w, e = select.select([sock], [], [], .0001) if not r: yield self.wait_on_self() additional_data += sock.recv(content_length) if len(additional_data) == content_length: break _log.debug( 'additional_data: {0}'.format(additional_data)) rest += additional_data body = rest _log.debug('body: {0}'.format(body)) option_names = ('method', 'uri', 'version', 'headers', 'remote_ip', 'content_length') options = dict(method=method, uri=uri, version=version, headers=headers, remote_ip=remote_ip, content_length=content_length, server_name=bind[0], port=bind[1]) _log.debug('options: {0}'.format(options)) _log.info( '{method} {uri} ({content_length})'.format(**options)) except Exception as exc: _log.error('Failed to parse request: {0}\n---\n{0}'.format( traceback.format_exc(), first_data)) message.reply(directive='respond', body={ 'response': 'HTTP/1.1 400 Invalid Request\r\n' 'Connection: close' }) break # Don't try to parse the request any further as # we've already replied with a 400 parsed = yield self.wait_on_message( to=self.id, directive='handle_request_body', body={ 'options': options, 'body': body }) response = yield self.wait_on_message( to=join_id('wsgi', self.hive.hive_id), directive='handle_request', body={ 'body': body, 'options': options, 'arguments': parsed.body['arguments'], 'files': parsed.body['files'] }) message.reply(directive='respond', body={'response': response.body.get('response')}) break
def send_the_message(self, message): self.hive.send_message(to=join_id(self.server_hive_id, self.server_hive_id), directive="greeting", body={"content": "hi, how are yuo"})