def run(self):
""" Loop forever sending data that is sent in on the queue """
# Set our handler to close gracefully
G.set_exit_handler(self.cleanup)
# Connect to remote host
self.connect()
if G.VERBOSE:
print "Starting RemoteQueue Client..."
# loop forever
while self.RUNNING:
# Get our data
try:
data = self.INPUT_QUEUE.get()
except:
if G.VERBOSE:
print "WARNING: Could not get data from queue!"
pass
if data == G.CTRL_CMD_KILL:
break
###
## TODO : Optimization!!!
###
# Extract index info
# machine = data['MACHINE']
# name = data['SUA_NAME']
# profile = data['SUA_PROFILE']
module = data['MODULE_NAME']
pb2_data = ProtoBuf.pack_sensor_output(data)
# Is this something new? If not lets not waste the bandwidth
if module not in self.cache or self.cache[module] != data:
self.cache[module] = data
else:
continue
# Try to write it to our socket
while True:
try:
G.send_socket_data(self.SOCK, pb2_data)
break
except:
if G.VERBOSE:
print "RemoteQueueClient: Socket Closed."
# Clear our cache and try to reconnect
del self.cache[module]
self.connect()
# Close our socket nicely
self.SOCK.close()
def run(self):
""" Loop forever sending data that is sent in on the queue """
# Set our handler to close gracefully
G.set_exit_handler(self.cleanup)
# Connect to remote host
self.connect()
if G.VERBOSE:
print "Starting RemoteQueue Client..."
# loop forever
while self.RUNNING:
# Get our data
try:
data = self.INPUT_QUEUE.get()
except:
if G.VERBOSE:
print "WARNING: Could not get data from queue!"
pass
if data == G.CTRL_CMD_KILL:
break
###
## TODO : Optimization!!!
###
# Extract index info
# machine = data['MACHINE']
# name = data['SUA_NAME']
# profile = data['SUA_PROFILE']
module = data['MODULE_NAME']
pb2_data = ProtoBuf.pack_sensor_output(data)
# Is this something new? If not lets not waste the bandwidth
if module not in self.cache or self.cache[module] != data:
self.cache[module] = data
else:
continue
# Try to write it to our socket
while True:
try:
G.send_socket_data(self.SOCK, pb2_data)
break
except:
if G.VERBOSE:
print "RemoteQueueClient: Socket Closed."
# Clear our cache and try to reconnect
del self.cache[module]
self.connect()
# Close our socket nicely
self.SOCK.close()
def run(self):
""" Loop until we fail relaying objects """
# Set our handler to close gracefully
G.set_exit_handler(self.cleanup)
if G.VERBOSE:
print "Relaying data from socket to queue."
while self.RUNNING:
# Try to unpack it
try:
# Get our data
data = G.read_socket_data(self.SOCK)
# Unpack our sensor output
data = ProtoBuf.unpack_sensor_output(data)
except EOFError:
if G.VERBOSE:
print "RemoteQueue: Looks like our socket closed."
break
except:
# Just die!
if self.RUNNING:
print "ERROR/RemoteQueue: Could not unpickle network data."
break
# update our machine name to indicate its origin
data['MACHINE'] = self.address[0] + "-" + data['MACHINE']
# Write the data to our queue, if we can
try:
self.OUTPUT_QUEUE.put(data, False)
except:
if self.RUNNING:
print "ERROR/RemoteQueue: Could not write to output queue."
G.print_traceback()
break
# Close socket
self.SOCK.close()
def run(self):
""" Loop until we fail relaying objects """
# Set our handler to close gracefully
G.set_exit_handler(self.cleanup)
if G.VERBOSE:
print "Relaying data from socket to queue."
while self.RUNNING:
# Try to unpack it
try:
# Get our data
data = G.read_socket_data(self.SOCK)
# Unpack our sensor output
data = ProtoBuf.unpack_sensor_output(data)
except EOFError:
if G.VERBOSE:
print "RemoteQueue: Looks like our socket closed."
break
except:
# Just die!
if self.RUNNING:
print "ERROR/RemoteQueue: Could not unpickle network data."
break
# update our machine name to indicate its origin
data['MACHINE'] = self.address[0] + "-" + data['MACHINE']
# Write the data to our queue, if we can
try:
self.OUTPUT_QUEUE.put(data, False)
except:
if self.RUNNING:
print "ERROR/RemoteQueue: Could not write to output queue."
G.print_traceback()
break
# Close socket
self.SOCK.close()
def run(self):
""" Bind a socket, and accept connections that send pickled objects """
# Set our handler to close gracefully
G.set_exit_handler(self.cleanup)
# Open our socket
self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Ignore the silly TIME_WAIT state
self.SOCK.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Bind to our address/port
BOUND = False
while not BOUND:
try:
self.SOCK.bind(G.QUEUE_ADDR)
BOUND = True
except:
print "RemoteQueue: Cannot bind socket... (Retrying in %d seconds)" % G.LOPHI_BIND_RETRY
time.sleep(G.LOPHI_BIND_RETRY)
# Listen for a client (Only 1 at a time)
self.SOCK.listen(2)
if G.VERBOSE:
print "RemoteQueue: Listening on %s:%s..." % (G.QUEUE_ADDR[0],
G.QUEUE_ADDR[1])
while self.RUNNING:
try:
clientsock, addr = self.SOCK.accept()
if G.VERBOSE:
print "RemoteQueue: Got connection from %s:%s." % (addr[0],
addr[1])
client = ClientRelay(clientsock, addr, self.OUTPUT_QUEUE)
client.start()
self.clients.append(client)
except:
break
if G.VERBOSE:
print "RemoteQueue: Closed"
def run(self):
""" Bind a socket, and accept connections that send pickled objects """
# Set our handler to close gracefully
G.set_exit_handler(self.cleanup)
# Open our socket
self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Ignore the silly TIME_WAIT state
self.SOCK.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Bind to our address/port
BOUND = False
while not BOUND:
try:
self.SOCK.bind(G.QUEUE_ADDR)
BOUND = True
except:
print "RemoteQueue: Cannot bind socket... (Retrying in %d seconds)" % G.LOPHI_BIND_RETRY
time.sleep(G.LOPHI_BIND_RETRY)
# Listen for a client (Only 1 at a time)
self.SOCK.listen(2)
if G.VERBOSE:
print "RemoteQueue: Listening on %s:%s..." % (G.QUEUE_ADDR[0], G.QUEUE_ADDR[1])
while self.RUNNING:
try:
clientsock, addr = self.SOCK.accept()
if G.VERBOSE:
print "RemoteQueue: Got connection from %s:%s." % (addr[0], addr[1])
client = ClientRelay(clientsock, addr, self.OUTPUT_QUEUE)
client.start()
self.clients.append(client)
except:
break
if G.VERBOSE:
print "RemoteQueue: Closed"
def start(self):
"""
Main function to just loop forever while waiting for input over amqp
"""
quit_commands = ['q', 'quit', 'exit']
# Setup our handler to close gracefully
G.set_exit_handler(self.cleanup)
# Setup or queues
self.manager = multiprocessing.Manager()
self.INPUT_QUEUE = self.manager.Queue()
# set of comm processes (rabbitmq, etc.) - for cleanup later
self.comm_processes = set([])
# Set up communication queue with all of our consumers, processes, and producers
self.in_queue = multiprocessing.Queue()
self.out_queue = multiprocessing.Queue()
# Listen for physical cards registering
# HOST = ''
# PORT = G.CARD_REG_PORT
#
# self.reg_consumer = Card_Reg_Server((HOST, PORT), UDPHandler, self.in_queue)
# self.reg_consumer.start()
# self.comm_processes.add(self.reg_consumer)
# Setup RabbitMQ consumers and queues
logger.debug("Starting up LOPHI RabbitmQ Producers...")
# self.ctrl_producer = rabbitmq.LOPHI_RPC_Producer(self.amqp_host,
# self.out_queue,
# G.RabbitMQ.CTRL_OUT,
# G.RabbitMQ.CTRL_IN,
# exchange_type=G.RabbitMQ.TYPE_FANOUT,
# exchange=G.RabbitMQ.EXCHANGE_FANOUT)
self.ctrl_producer = rabbitmq.LOPHI_RabbitMQ_Producer(
self.amqp_host,
self.out_queue,
G.RabbitMQ.CTRL_OUT,
exchange_type=G.RabbitMQ.TYPE_FANOUT,
routing_key='',
exchange=G.RabbitMQ.EXCHANGE_FANOUT)
self.ctrl_producer.start()
self.comm_processes.add(self.ctrl_producer)
logger.debug("Starting up LOPHI RabbitMQ Consumers...")
# Listen for control messages, e.g. from a CLI
# self.ctrl_consumer = rabbitmq.LOPHI_RPC_Consumer(self.amqp_host,
# self.in_queue,
# G.RabbitMQ.CTRL_IN)
self.ctrl_consumer = rabbitmq.LOPHI_RabbitMQ_Consumer(
self.amqp_host, self.in_queue, G.RabbitMQ.CTRL_IN)
self.ctrl_consumer.start()
self.comm_processes.add(self.ctrl_consumer)
# Connect to all of our controllers
for c in self.config_list:
self.config_list[c].connect()
print "Waiting for input from queues."
# Just loop forever taking input from rabbitmq
while 1:
user_input = self.in_queue.get()
# Decode input from rabbitmq format
try:
# (corr_id, routing_key, msg) = json.loads(user_input)
# type is type of message
# command
# (type, cmd_data) = msg
cmd = LophiCommand.from_data(user_input)
except:
print "Unknown command: ", user_input
continue
logger.debug("Received msg %s" % cmd)
# check if type is valid
# if msg.type not in self.MSG_TYPES:
# print "Invalid message type: %s\n" % type
# See if it's valid command
if cmd.cmd not in self.COMMANDS.keys():
resp = "Invalid Command: %s\n" % cmd.cmd
logger.debug(resp)
response = cmd.make_response(resp)
self.out_queue.put(response)
else:
# self.process_cmd(type, cmd, corr_id, routing_key)
self.process_cmd(cmd)
"""
@todo: add command to kill master
"""
# Call our cleanup function and shutdown nicely
self.cleanup(None)
def start(self):
"""
Main function to just loop forever while waiting for input over amqp
"""
quit_commands = ['q', 'quit', 'exit']
# Setup our handler to close gracefully
G.set_exit_handler(self.cleanup)
# Setup or queues
self.manager = multiprocessing.Manager()
self.INPUT_QUEUE = self.manager.Queue()
# set of comm processes (rabbitmq, etc.) - for cleanup later
self.comm_processes = set([])
# Set up communication queue with all of our consumers, processes, and producers
self.in_queue = multiprocessing.Queue()
self.out_queue = multiprocessing.Queue()
# Listen for physical cards registering
# HOST = ''
# PORT = G.CARD_REG_PORT
#
# self.reg_consumer = Card_Reg_Server((HOST, PORT), UDPHandler, self.in_queue)
# self.reg_consumer.start()
# self.comm_processes.add(self.reg_consumer)
# Setup RabbitMQ consumers and queues
logger.debug("Starting up LOPHI RabbitmQ Producers...")
# self.ctrl_producer = rabbitmq.LOPHI_RPC_Producer(self.amqp_host,
# self.out_queue,
# G.RabbitMQ.CTRL_OUT,
# G.RabbitMQ.CTRL_IN,
# exchange_type=G.RabbitMQ.TYPE_FANOUT,
# exchange=G.RabbitMQ.EXCHANGE_FANOUT)
self.ctrl_producer = rabbitmq.LOPHI_RabbitMQ_Producer(self.amqp_host,
self.out_queue,
G.RabbitMQ.CTRL_OUT,
exchange_type=G.RabbitMQ.TYPE_FANOUT,
routing_key='',
exchange=G.RabbitMQ.EXCHANGE_FANOUT)
self.ctrl_producer.start()
self.comm_processes.add(self.ctrl_producer)
logger.debug("Starting up LOPHI RabbitMQ Consumers...")
# Listen for control messages, e.g. from a CLI
# self.ctrl_consumer = rabbitmq.LOPHI_RPC_Consumer(self.amqp_host,
# self.in_queue,
# G.RabbitMQ.CTRL_IN)
self.ctrl_consumer = rabbitmq.LOPHI_RabbitMQ_Consumer(self.amqp_host,
self.in_queue,
G.RabbitMQ.CTRL_IN)
self.ctrl_consumer.start()
self.comm_processes.add(self.ctrl_consumer)
# Connect to all of our controllers
for c in self.config_list:
self.config_list[c].connect()
print "Waiting for input from queues."
# Just loop forever taking input from rabbitmq
while 1:
user_input = self.in_queue.get()
# Decode input from rabbitmq format
try:
# (corr_id, routing_key, msg) = json.loads(user_input)
# type is type of message
# command
# (type, cmd_data) = msg
cmd = LophiCommand.from_data(user_input)
except:
print "Unknown command: ", user_input
continue
logger.debug("Received msg %s" % cmd)
# check if type is valid
# if msg.type not in self.MSG_TYPES:
# print "Invalid message type: %s\n" % type
# See if it's valid command
if cmd.cmd not in self.COMMANDS.keys():
resp = "Invalid Command: %s\n" % cmd.cmd
logger.debug(resp)
response = cmd.make_response(resp)
self.out_queue.put(response)
else:
# self.process_cmd(type, cmd, corr_id, routing_key)
self.process_cmd(cmd)
"""
@todo: add command to kill master
"""
# Call our cleanup function and shutdown nicely
self.cleanup(None)
