def run(self): """ Loop forever sending data that is sent in on the queue """ # Set our handler to close gracefully G.set_exit_handler(self.cleanup) # Connect to remote host self.connect() if G.VERBOSE: print "Starting RemoteQueue Client..." # loop forever while self.RUNNING: # Get our data try: data = self.INPUT_QUEUE.get() except: if G.VERBOSE: print "WARNING: Could not get data from queue!" pass if data == G.CTRL_CMD_KILL: break ### ## TODO : Optimization!!! ### # Extract index info # machine = data['MACHINE'] # name = data['SUA_NAME'] # profile = data['SUA_PROFILE'] module = data['MODULE_NAME'] pb2_data = ProtoBuf.pack_sensor_output(data) # Is this something new? If not lets not waste the bandwidth if module not in self.cache or self.cache[module] != data: self.cache[module] = data else: continue # Try to write it to our socket while True: try: G.send_socket_data(self.SOCK, pb2_data) break except: if G.VERBOSE: print "RemoteQueueClient: Socket Closed." # Clear our cache and try to reconnect del self.cache[module] self.connect() # Close our socket nicely self.SOCK.close()
def run(self): """ Loop until we fail relaying objects """ # Set our handler to close gracefully G.set_exit_handler(self.cleanup) if G.VERBOSE: print "Relaying data from socket to queue." while self.RUNNING: # Try to unpack it try: # Get our data data = G.read_socket_data(self.SOCK) # Unpack our sensor output data = ProtoBuf.unpack_sensor_output(data) except EOFError: if G.VERBOSE: print "RemoteQueue: Looks like our socket closed." break except: # Just die! if self.RUNNING: print "ERROR/RemoteQueue: Could not unpickle network data." break # update our machine name to indicate its origin data['MACHINE'] = self.address[0] + "-" + data['MACHINE'] # Write the data to our queue, if we can try: self.OUTPUT_QUEUE.put(data, False) except: if self.RUNNING: print "ERROR/RemoteQueue: Could not write to output queue." G.print_traceback() break # Close socket self.SOCK.close()
def run(self): """ Bind a socket, and accept connections that send pickled objects """ # Set our handler to close gracefully G.set_exit_handler(self.cleanup) # Open our socket self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Ignore the silly TIME_WAIT state self.SOCK.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Bind to our address/port BOUND = False while not BOUND: try: self.SOCK.bind(G.QUEUE_ADDR) BOUND = True except: print "RemoteQueue: Cannot bind socket... (Retrying in %d seconds)" % G.LOPHI_BIND_RETRY time.sleep(G.LOPHI_BIND_RETRY) # Listen for a client (Only 1 at a time) self.SOCK.listen(2) if G.VERBOSE: print "RemoteQueue: Listening on %s:%s..." % (G.QUEUE_ADDR[0], G.QUEUE_ADDR[1]) while self.RUNNING: try: clientsock, addr = self.SOCK.accept() if G.VERBOSE: print "RemoteQueue: Got connection from %s:%s." % (addr[0], addr[1]) client = ClientRelay(clientsock, addr, self.OUTPUT_QUEUE) client.start() self.clients.append(client) except: break if G.VERBOSE: print "RemoteQueue: Closed"
def start(self): """ Main function to just loop forever while waiting for input over amqp """ quit_commands = ['q', 'quit', 'exit'] # Setup our handler to close gracefully G.set_exit_handler(self.cleanup) # Setup or queues self.manager = multiprocessing.Manager() self.INPUT_QUEUE = self.manager.Queue() # set of comm processes (rabbitmq, etc.) - for cleanup later self.comm_processes = set([]) # Set up communication queue with all of our consumers, processes, and producers self.in_queue = multiprocessing.Queue() self.out_queue = multiprocessing.Queue() # Listen for physical cards registering # HOST = '' # PORT = G.CARD_REG_PORT # # self.reg_consumer = Card_Reg_Server((HOST, PORT), UDPHandler, self.in_queue) # self.reg_consumer.start() # self.comm_processes.add(self.reg_consumer) # Setup RabbitMQ consumers and queues logger.debug("Starting up LOPHI RabbitmQ Producers...") # self.ctrl_producer = rabbitmq.LOPHI_RPC_Producer(self.amqp_host, # self.out_queue, # G.RabbitMQ.CTRL_OUT, # G.RabbitMQ.CTRL_IN, # exchange_type=G.RabbitMQ.TYPE_FANOUT, # exchange=G.RabbitMQ.EXCHANGE_FANOUT) self.ctrl_producer = rabbitmq.LOPHI_RabbitMQ_Producer( self.amqp_host, self.out_queue, G.RabbitMQ.CTRL_OUT, exchange_type=G.RabbitMQ.TYPE_FANOUT, routing_key='', exchange=G.RabbitMQ.EXCHANGE_FANOUT) self.ctrl_producer.start() self.comm_processes.add(self.ctrl_producer) logger.debug("Starting up LOPHI RabbitMQ Consumers...") # Listen for control messages, e.g. from a CLI # self.ctrl_consumer = rabbitmq.LOPHI_RPC_Consumer(self.amqp_host, # self.in_queue, # G.RabbitMQ.CTRL_IN) self.ctrl_consumer = rabbitmq.LOPHI_RabbitMQ_Consumer( self.amqp_host, self.in_queue, G.RabbitMQ.CTRL_IN) self.ctrl_consumer.start() self.comm_processes.add(self.ctrl_consumer) # Connect to all of our controllers for c in self.config_list: self.config_list[c].connect() print "Waiting for input from queues." # Just loop forever taking input from rabbitmq while 1: user_input = self.in_queue.get() # Decode input from rabbitmq format try: # (corr_id, routing_key, msg) = json.loads(user_input) # type is type of message # command # (type, cmd_data) = msg cmd = LophiCommand.from_data(user_input) except: print "Unknown command: ", user_input continue logger.debug("Received msg %s" % cmd) # check if type is valid # if msg.type not in self.MSG_TYPES: # print "Invalid message type: %s\n" % type # See if it's valid command if cmd.cmd not in self.COMMANDS.keys(): resp = "Invalid Command: %s\n" % cmd.cmd logger.debug(resp) response = cmd.make_response(resp) self.out_queue.put(response) else: # self.process_cmd(type, cmd, corr_id, routing_key) self.process_cmd(cmd) """ @todo: add command to kill master """ # Call our cleanup function and shutdown nicely self.cleanup(None)
def start(self): """ Main function to just loop forever while waiting for input over amqp """ quit_commands = ['q', 'quit', 'exit'] # Setup our handler to close gracefully G.set_exit_handler(self.cleanup) # Setup or queues self.manager = multiprocessing.Manager() self.INPUT_QUEUE = self.manager.Queue() # set of comm processes (rabbitmq, etc.) - for cleanup later self.comm_processes = set([]) # Set up communication queue with all of our consumers, processes, and producers self.in_queue = multiprocessing.Queue() self.out_queue = multiprocessing.Queue() # Listen for physical cards registering # HOST = '' # PORT = G.CARD_REG_PORT # # self.reg_consumer = Card_Reg_Server((HOST, PORT), UDPHandler, self.in_queue) # self.reg_consumer.start() # self.comm_processes.add(self.reg_consumer) # Setup RabbitMQ consumers and queues logger.debug("Starting up LOPHI RabbitmQ Producers...") # self.ctrl_producer = rabbitmq.LOPHI_RPC_Producer(self.amqp_host, # self.out_queue, # G.RabbitMQ.CTRL_OUT, # G.RabbitMQ.CTRL_IN, # exchange_type=G.RabbitMQ.TYPE_FANOUT, # exchange=G.RabbitMQ.EXCHANGE_FANOUT) self.ctrl_producer = rabbitmq.LOPHI_RabbitMQ_Producer(self.amqp_host, self.out_queue, G.RabbitMQ.CTRL_OUT, exchange_type=G.RabbitMQ.TYPE_FANOUT, routing_key='', exchange=G.RabbitMQ.EXCHANGE_FANOUT) self.ctrl_producer.start() self.comm_processes.add(self.ctrl_producer) logger.debug("Starting up LOPHI RabbitMQ Consumers...") # Listen for control messages, e.g. from a CLI # self.ctrl_consumer = rabbitmq.LOPHI_RPC_Consumer(self.amqp_host, # self.in_queue, # G.RabbitMQ.CTRL_IN) self.ctrl_consumer = rabbitmq.LOPHI_RabbitMQ_Consumer(self.amqp_host, self.in_queue, G.RabbitMQ.CTRL_IN) self.ctrl_consumer.start() self.comm_processes.add(self.ctrl_consumer) # Connect to all of our controllers for c in self.config_list: self.config_list[c].connect() print "Waiting for input from queues." # Just loop forever taking input from rabbitmq while 1: user_input = self.in_queue.get() # Decode input from rabbitmq format try: # (corr_id, routing_key, msg) = json.loads(user_input) # type is type of message # command # (type, cmd_data) = msg cmd = LophiCommand.from_data(user_input) except: print "Unknown command: ", user_input continue logger.debug("Received msg %s" % cmd) # check if type is valid # if msg.type not in self.MSG_TYPES: # print "Invalid message type: %s\n" % type # See if it's valid command if cmd.cmd not in self.COMMANDS.keys(): resp = "Invalid Command: %s\n" % cmd.cmd logger.debug(resp) response = cmd.make_response(resp) self.out_queue.put(response) else: # self.process_cmd(type, cmd, corr_id, routing_key) self.process_cmd(cmd) """ @todo: add command to kill master """ # Call our cleanup function and shutdown nicely self.cleanup(None)