def test_peek_one_item_one_queue(self):
value = 5
queue = FifoQueue()
queue.enqueue(value)
peeked_value = queue.peek()
self.assertEqual(1, queue.size)
self.assertEqual(5, peeked_value)
def __init__(self, port, bridge_port=None,
queue_in_size=DEFAULT_IN_SIZE,
queue_out_size=DEFAULT_OUT_SIZE):
"""
Initialize.
port specifies the IP port to listen on. bridge_port is used
for briding to an existing serial-forward (such as that
provided by sim-sf).
queue_in_size and queue_out_size limit the maximum number of
queued elements. A size of zero specified that there is no
limit. When the queues fill up a warning is generated and the
oldest element is dropped.
"""
asyncore.dispatcher.__init__(self)
# maintained with callbacks from InjectSock, etc.
self.map = {}
self.upstream = []
self.bridge_source = None
# of (client, packet)
# where client is the source, or None if from the proxy source
self.q_in = FifoQueue(queue_in_size)
# of (target, packet)
# where target is a client to send to, or None to send to all
self.q_out = FifoQueue(queue_out_size)
self._init_server(("", port))
self._init_trigger(("127.0.0.1", port))
# if bridge_port is not None:
# self._init_bridge(("127.0.0.1", bridge_port))
self.thread = None
def test_dequeue_one_item_on_queue(self):
value = 5
queue = FifoQueue()
queue.enqueue(value)
removed_item = queue.dequeue()
self.assertEqual(0, queue.size)
self.assertEqual(5, removed_item)
def test_enqueue_two_items(self):
value1, value2 = 5, 10
queue = FifoQueue()
queue.enqueue(value1)
queue.enqueue(value2)
self.assertEqual(2, queue.size)
self.assertEqual(5, queue.ll.head.value)
self.assertEqual(10, queue.ll.tail.value)
def trade_ledger(self):
instrument_queues = {}
order_pairs = {}
for o in self.orders:
if o.instrument not in instrument_queues:
instrument_queues[o.instrument] = (FifoQueue(), FifoQueue()
) # (longs, shorts)
longs, shorts = instrument_queues[o.instrument]
if o.quantity < 0:
shorts.put(o)
else:
longs.put(o)
# match up the longs and shorts
for inst in instrument_queues.keys():
longs, shorts = instrument_queues[inst]
open_close_order_pairs = []
while not longs.empty() and not shorts.empty():
long, short = longs.get(), shorts.get()
open_order, close_order = (long, short) if long.execution_end_time() < short.execution_end_time() \
else (short, long)
open_close_order_pairs.append((open_order, close_order))
# handle unmatched longs or shorts i.e. positions currently open
while not longs.empty() or not shorts.empty():
unclosed_open_order = longs.get(
) if not longs.empty() else shorts.get()
open_close_order_pairs.append((unclosed_open_order, None))
order_pairs[inst] = open_close_order_pairs
trade_df = []
for inst in order_pairs.keys():
for open_order, close_order in [(o, c)
for o, c in order_pairs[inst]]:
if close_order:
end_dt = close_order.execution_end_time()
end_value = close_order.executed_price
status = 'closed'
else:
end_dt = None
end_value = None
status = 'open'
start_dt, open_value = open_order.execution_end_time(
), open_order.executed_price
long_or_short = np.sign(open_order.quantity)
trade_df.append(
(inst, start_dt, end_dt, open_value, end_value,
long_or_short, status, (end_value - open_value) *
long_or_short if status == 'closed' else None))
return pd.DataFrame(trade_df,
columns=[
'Instrument', 'Open', 'Close', 'Open Value',
'Close Value', 'Long Short', 'Status',
'Trade PnL'
])
def __init__(self,
port=None,
queue_in_size=DEFAULT_IN_SIZE,
queue_out_size=DEFAULT_OUT_SIZE):
"""
Initialize.
port specifies the IP port to listen on.
queue_in_size and queue_out_size limit the maximum number of
queued elements. A size of zero specifies that there is no
limit. When the queues fill up a warning is generated and the
oldest element is dropped.
"""
asyncore.dispatcher.__init__(self)
self.connection_map = {} # connection => connection
self.clients = [] # connections that want messages
self.server_map = ServerMap()
self.trigger = None
self.trigger_magic = None # must be matched to accept trigger
# of (client, message, handler)
self.q_in = FifoQueue(queue_in_size)
# of (target, packet)
# where target is a client to send to, or None to send to all
self.q_out = FifoQueue(queue_out_size)
# PST: could possibly use UNIX sockets when not windows
self._init_server(("", port))
self._init_trigger(("127.0.0.1", port))
self.thread = None
# setup basic handlers
def message_rejected(client, message):
print "MessageRejected(Dummy Handler) {%s}" % message
def mapped_name(client, message):
print "MappedName: %s -> %d" % (message.name, message.id)
client.map.add_resolution(message.name, message.id)
self.server_map.set_handler(MessageRejected, message_rejected)
self.server_map.set_handler(MappedName, mapped_name)
def test_peek_multiple_items_on_queue(self):
value1, value2, value3 = 5, 10, 15
queue = FifoQueue()
queue.enqueue(value1)
queue.enqueue(value2)
queue.enqueue(value3)
peeked_item = queue.peek()
self.assertEqual(3, queue.size)
self.assertEqual(5, peeked_item)
def breadth_first_search(self):
visited_nodes = []
# Use a queue to keep track of all the nodes you need to visit
node_queue = FifoQueue()
# Set starting point as no root node
current_node = self.edges[0].node_from
visited_nodes.append(current_node)
while current_node:
new_edge = self._return_new_edge(current_node, visited_nodes)
if new_edge:
visited_nodes.append(new_edge.node_to)
node_queue.enqueue(new_edge.node_to)
else:
current_node = node_queue.dequeue()
visited_node_values = [node.value for node in visited_nodes]
return visited_node_values
def test_dequeue_multiple_items_on_queue(self):
value1, value2, value3 = 5, 10, 15
queue = FifoQueue()
queue.enqueue(value1)
queue.enqueue(value2)
queue.enqueue(value3)
removed_item1 = queue.dequeue()
removed_item2 = queue.dequeue()
removed_item3 = queue.dequeue()
self.assertEqual(0, queue.size)
self.assertEqual(5, removed_item1)
self.assertEqual(10, removed_item2)
self.assertEqual(15, removed_item3)
def __init__(self, port=None, queue_in_size=DEFAULT_IN_SIZE, queue_out_size=DEFAULT_OUT_SIZE):
"""
Initialize.
port specifies the IP port to listen on.
queue_in_size and queue_out_size limit the maximum number of
queued elements. A size of zero specifies that there is no
limit. When the queues fill up a warning is generated and the
oldest element is dropped.
"""
asyncore.dispatcher.__init__(self)
self.connection_map = {} # connection => connection
self.clients = [] # connections that want messages
self.server_map = ServerMap()
self.trigger = None
self.trigger_magic = None # must be matched to accept trigger
# of (client, message, handler)
self.q_in = FifoQueue(queue_in_size)
# of (target, packet)
# where target is a client to send to, or None to send to all
self.q_out = FifoQueue(queue_out_size)
# PST: could possibly use UNIX sockets when not windows
self._init_server(("", port))
self._init_trigger(("127.0.0.1", port))
self.thread = None
# setup basic handlers
def message_rejected(client, message):
print "MessageRejected(Dummy Handler) {%s}" % message
def mapped_name(client, message):
print "MappedName: %s -> %d" % (message.name, message.id)
client.map.add_resolution(message.name, message.id)
self.server_map.set_handler(MessageRejected, message_rejected)
self.server_map.set_handler(MappedName, mapped_name)
def test_enqueue_three_items(self):
value1, value2, value3 = 5, 10, 15
queue = FifoQueue()
queue.enqueue(value1)
queue.enqueue(value2)
queue.enqueue(value3)
self.assertEqual(3, queue.size)
self.assertEqual(5, queue.ll.head.value)
self.assertEqual(15, queue.ll.tail.value)
class Scheduler:
def __init__(self):
self.fifo_queue = FifoQueue()
self.priority_queue = PriorityQueue()
# enfileira processo se a fila nao estiver lotada (1000)
def queueProcess(self, process):
if (process.priority == 0):
result = self.fifo_queue.push(process)
else:
result = self.priority_queue.push(process)
return result
# escalona o processo com maior prioridade para a CPU
def scheduleProcess(self):
if (not self.fifo_queue.is_empty()):
process = self.fifo_queue.pop()
else:
process = self.priority_queue.pop()
return process
def preemptProcess(self, process, globalTime):
if (process.priority > 0): #se o processo eh de usuário
if ((not self.fifo_queue.is_empty())
and (globalTime >= self.fifo_queue.pick().timeOfArrival)
): # se chegou um proc de tempo real
if (process.priority > 1): # se a prioridade ja nao eh máxima
process.priority -= 1 # aumenta a prioridade
self.priority_queue.push(
process) # volta pra fila com sua respectiva prioridade
process = self.fifo_queue.pop(
) # pega o processo de tempo real prioritario
elif ((not self.priority_queue.is_empty())
and (globalTime >= self.priority_queue.pick().timeOfArrival)
and
(process.priority > self.priority_queue.pick().priority)):
if (process.priority > 1): # se a prioridade ja nao eh máxima
process.priority -= 1 # aumenta a prioridade
self.priority_queue.push(
process) # volta pra fila com sua respectiva prioridade
process = self.priority_queue.pop(
) # pega o processo de usuario prioritario
return process
class Inject(asyncore.dispatcher):
"""
Select-dispatch to move packets across sf-compatible streams.
"""
log = logging.getLogger(__name__)
def __init__(self, port, bridge_port=None,
queue_in_size=DEFAULT_IN_SIZE,
queue_out_size=DEFAULT_OUT_SIZE):
"""
Initialize.
port specifies the IP port to listen on. bridge_port is used
for briding to an existing serial-forward (such as that
provided by sim-sf).
queue_in_size and queue_out_size limit the maximum number of
queued elements. A size of zero specified that there is no
limit. When the queues fill up a warning is generated and the
oldest element is dropped.
"""
asyncore.dispatcher.__init__(self)
# maintained with callbacks from InjectSock, etc.
self.map = {}
self.upstream = []
self.bridge_source = None
# of (client, packet)
# where client is the source, or None if from the proxy source
self.q_in = FifoQueue(queue_in_size)
# of (target, packet)
# where target is a client to send to, or None to send to all
self.q_out = FifoQueue(queue_out_size)
self._init_server(("", port))
self._init_trigger(("127.0.0.1", port))
# if bridge_port is not None:
# self._init_bridge(("127.0.0.1", bridge_port))
self.thread = None
def _init_server(self, server_addr):
"""
Setup server socket.
"""
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.bind(server_addr)
self.listen(CONNECTION_QUEUE_SIZE)
self.map[self] = self
def _init_bridge(self, bridge_addr):
"""
Setup bridge socket.
"""
source = InjectDownstream(sf_inject=self)
source.create_socket(socket.AF_INET, socket.SOCK_STREAM)
source.connect(bridge_addr)
# sim-sf will block until it receives the serial-forward
# version and, since it (sf.process()) does not perform a GIL
# release, this thread will never resume. This will ensure
# that the sim-sf serial forwarder doesn't hang (here) by
# forcing data onto the wire.
source.send(SFCLIENT_MAGIC)
self.bridge_source = source
def _init_trigger(self, server_addr):
"""
Setup trigger socket.
The trigger is used to interrupt the asyncore loop and force
examination of the message queue.
"""
trigger = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
trigger.connect(server_addr)
trigger.shutdown(socket.SHUT_RD)
# How the handle_accept identifies this as a trigger. Very
# important to prevent blocking with handle_accept+sync_read.
trigger.send(TRIGGER_MAGIC)
self.trigger = trigger
def packet_read(self, conn, packet):
"""
Read in a packet and dispatch -- asyncore callback.
"""
if conn == self.bridge_source:
# TODO: fix (huh?)
self.send_packet_upstream(packet)
else:
self.q_in.fifo_put((conn, packet))
# foward to motes (should be filtered)
if self.bridge_source:
self.bridge_source.write_packet(packet)
def handle_accept(self):
"""
Process new connections (sf-client or trigger) -- asyncore
callback.
"""
client, addr = self.accept()
# Determine who connected and if they look valid. This is done
# first to siphon off the 'trigger' handler. It also blocks.
magic = sync_read(client, 2)
if is_trigger(magic):
InjectTrigger(sf_inject=self, socket=client)
elif is_sfclient(magic):
InjectUpstream(sf_inject=self, socket=client)
client.send(SFCLIENT_MAGIC)
else:
warn(InvalidVersionWarning())
client.close()
def attach_conn(self, conn):
"""
Attach a connections.
"""
if not conn in self.map:
if isinstance(conn, InjectUpstream):
self.upstream.append(conn)
self.map[conn] = conn
def detach_conn(self, conn):
"""
Remove a connection.
"""
if conn in self.upstream:
self.upstream.remove(conn)
del(self.map[conn])
#
# Internal
#
def _message_to_packet(self, msg, dest=0xFFFF, group=0xAA):
"""
Wraps a message in an AM (Serial) packet.
"""
payload = msg.dataGet()
am_type = msg.amType()
pkt = SerialPacket(None)
pkt.set_header_dest(dest)
pkt.set_header_group(group)
pkt.set_header_type(int(am_type))
pkt.set_header_length(len(payload))
# first byte, always 0, identifies as AM packet
return chr(0) + pkt.dataGet()[0:pkt.offset_data(0)] + payload
def _send_queued_packets(self):
"""
Flush all queued out packets onto the wire.
"""
while not self.q_out.empty():
(client, message) = self.q_out.get()
serial_packet = self._message_to_packet(message)
if client is not None:
# send to specific connection
client.write_packet(serial_packet)
else:
# send to all upstream connections
for conn in self.upstream:
conn.write_packet(serial_packet)
def _thread_pump(self):
"""
Pump that runs inside the dispatch thread. This never normally
terminates.
"""
self.log.debug("Pump started")
while True:
asyncore.loop(SELECT_IDLE_SECONDS, False, self.map, 1)
self._send_queued_packets()
#
# Public
#
def read_packets(self):
"""
Returns a list of (client, packet) for all incoming packets.
The packets are removed from the incoming queue and have the
AM (Serial) header stripped.
"""
read = []
while not self.q_in.empty():
(client, data) = self.q_in.get()
read.append((client, data[8:]))
return read
def inject(self, message, target=None):
"""
Inject a message.
The message is sent to target. If target is None the message
is sent to everyone. This is safe to call outside the event
thread.
"""
self.q_out.fifo_put((target, message))
self.trigger.send("t")
def start(self):
"""
Spawn a thread and start running the injection bridge.
Returns self.
"""
def launcher():
"Terminate on pump exception"
try:
self._thread_pump()
except:
traceback.print_exc()
print >>sys.stderr, "!!! INJECTOR TERMINATED !!!"
os._exit(1)
self.thread = thread.start_new_thread(launcher, ())
return self
def __init__(self):
self.fifo_queue = FifoQueue()
self.priority_queue = PriorityQueue()
class ProtoServer(asyncore.dispatcher):
"""
Server for ProtoMap.
"""
log = logging.getLogger(__name__)
def __init__(self, port=None, queue_in_size=DEFAULT_IN_SIZE, queue_out_size=DEFAULT_OUT_SIZE):
"""
Initialize.
port specifies the IP port to listen on.
queue_in_size and queue_out_size limit the maximum number of
queued elements. A size of zero specifies that there is no
limit. When the queues fill up a warning is generated and the
oldest element is dropped.
"""
asyncore.dispatcher.__init__(self)
self.connection_map = {} # connection => connection
self.clients = [] # connections that want messages
self.server_map = ServerMap()
self.trigger = None
self.trigger_magic = None # must be matched to accept trigger
# of (client, message, handler)
self.q_in = FifoQueue(queue_in_size)
# of (target, packet)
# where target is a client to send to, or None to send to all
self.q_out = FifoQueue(queue_out_size)
# PST: could possibly use UNIX sockets when not windows
self._init_server(("", port))
self._init_trigger(("127.0.0.1", port))
self.thread = None
# setup basic handlers
def message_rejected(client, message):
print "MessageRejected(Dummy Handler) {%s}" % message
def mapped_name(client, message):
print "MappedName: %s -> %d" % (message.name, message.id)
client.map.add_resolution(message.name, message.id)
self.server_map.set_handler(MessageRejected, message_rejected)
self.server_map.set_handler(MappedName, mapped_name)
def _init_server(self, server_addr):
"""
Setup server socket.
"""
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.bind(server_addr)
self.listen(CONNECTION_QUEUE_SIZE)
self.connection_map[self] = self
def _init_trigger(self, server_addr):
"""
Setup trigger socket.
The trigger is used to interrupt the asyncore loop and force
examination of the outbound message queue. (This is because
``threading'' in Python 2.6 and before is absolutely soupy.)
"""
assert not self.trigger
trigger = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
trigger.connect(server_addr)
trigger.shutdown(socket.SHUT_RD)
# to identify as trigger
self.trigger_magic = "TRIGGER:%08x" % random.randint(0, 0xFFFFFFFF)
trigger.send(self.trigger_magic)
self.trigger = trigger
def packet_read(self, conn, raw_packet):
"""
Read in a packet and dispatch -- asyncore callback.
"""
packet = Packet.decode(raw_packet)
print "packet in with protocol"
if Packet.is_named(packet.protocol):
# protocol is named, resolve id first
name = packet.protocol
id = self.server_map.resolve_id(name)
if id:
named = MappedName()
named.name = name
named.id = id
msg = Message(0, named)
self.q_out.fifo_put((conn, msg))
else:
print "unmapped name: " + str(name)
rejected = MessageRejected()
rejected.id = 0
rejected.reason = rejected.UNMAPPED_NAME
rejected.message = "Unmapped name: '%s'" % name
msg = Message(0, rejected)
self.q_out.fifo_put((conn, msg))
# stop processing
return
else:
# just take id
id = packet.protocol
# this seems slightly silly in the case where the message was
# a named message (it shouldn't hurt though)
name = self.server_map.resolve_name(id)
if not name:
# if the reverse name lookup fails
print "unmapped id: " + str(id)
rejected = MessageRejected()
rejected.id = 0
rejected.reason = rejected.UNMAPPED_ID
rejected.message = "Unmapped ID: %d" % id
msg = Message(0, rejected)
self.q_out.fifo_put((conn, msg))
# stop processing
return
protobuf_class = self.server_map.resolve_protobuffer(name)
assert protobuf_class
protobuf = protobuf_class()
protobuf.ParseFromString(packet.payload)
message = Message(packet.track_id, protobuf, name=name)
print "have protobuf: {" + str(protobuf) + "}"
self.q_in.fifo_put((conn, message))
def handle_accept(self):
"""
Process new connections (asyncore callback).
"""
client, addr = self.accept()
holding = Holding(PROTOMAP_MAGIC, self, client, self.trigger_magic)
self.attach_connection(holding)
def attach_connection(self, new_conn):
"""
Attach a new connection.
"""
self.alter_connection(None, new_conn)
def alter_connection(self, old_conn, new_conn):
"""
Alter an existing connection handler; if new_conn is None the
connection is closed. The old_conn, if any, is removed.
"""
if old_conn and new_conn:
# prevent socket-swapping
assert old_conn.socket is new_conn.socket
# remove existing (if any)
if old_conn in self.connection_map:
del self.connection_map[old_conn]
if old_conn in self.clients:
self.clients.remove(old_conn)
# attach new (if any)
if new_conn:
self.connection_map[new_conn] = new_conn
if isinstance(new_conn, ProtoClient):
self.clients.append(new_conn)
elif old_conn:
# no new_conn, but old_conn -- close connection
old_conn.close()
def connection_closed(self, conn):
"""
Remove a connection.
"""
self.log.debug("Connection closed")
if conn in self.clients:
self.clients.remove(conn)
del self.connection_map[conn]
#
# Internal
#
# PST- can avoid this and keep asyncore loop tighter?
# PST- could possibly send packet to closed connection
def _send_queued_packets(self):
"""
Flush all queued out packets onto the wire.
"""
def send(client, message):
protocol = client.map.resolve_id(message.name) or message.name
packet = message.encode(protocol)
print "writing packet " + str(protocol)
client.write_packet(packet)
while not self.q_out.empty():
client, message = self.q_out.get()
if client:
send(client, message)
else:
for client in self.clients:
send(client, message)
def _thread_pump(self):
"""
Pump that runs inside the dispatch thread. This never normally
terminates.
"""
self.log.debug("Pump started")
while True:
asyncore.loop(SELECT_IDLE_SECONDS, False, self.connection_map, 1)
self._send_queued_packets()
#
# Public
#
def process_messages(self):
"""
Invoke handlers for messages in the incoming queue.
This method should be invoked in the thread where the handlers
are to be executed.
"""
while not self.q_in.empty():
client, message = self.q_in.get()
handler = self.server_map.resolve_handler(message.name)
handler(client, message)
def write(self, message, target=None):
"""
Write a message.
The message is sent to target. If target is None the message
is sent to all connected clients. This method is safe to call
outside the event thread.
"""
self.q_out.fifo_put((target, message))
self.trigger.send("t")
def start(self):
"""
Spawn a thread and start running the server.
Returns self.
"""
def launcher():
"Terminate on pump exception"
try:
self._thread_pump()
except:
traceback.print_exc()
print >> sys.stderr, "!!! ProtoServer Terminated !!!"
os._exit(1)
self.thread = thread.start_new_thread(launcher, ())
return self
def test_peek_no_items_on_queue(self):
queue = FifoQueue()
peeked_value = queue.peek()
self.assertEqual(0, queue.size)
self.assertEqual(None, peeked_value)
def test_queue_create(self):
queue = FifoQueue()
self.assertEqual(0, queue.size)
def test_enqueue_one_item(self):
value = 5
queue = FifoQueue()
queue.enqueue(value)
self.assertEqual(1, queue.size)
self.assertEqual(5, queue.ll.head.value)
def test_dequeue_no_items_on_queue(self):
queue = FifoQueue()
removed_item = queue.dequeue()
self.assertEqual(0, queue.size)
self.assertEqual(None, removed_item)
class ProtoServer(asyncore.dispatcher):
"""
Server for ProtoMap.
"""
log = logging.getLogger(__name__)
def __init__(self,
port=None,
queue_in_size=DEFAULT_IN_SIZE,
queue_out_size=DEFAULT_OUT_SIZE):
"""
Initialize.
port specifies the IP port to listen on.
queue_in_size and queue_out_size limit the maximum number of
queued elements. A size of zero specifies that there is no
limit. When the queues fill up a warning is generated and the
oldest element is dropped.
"""
asyncore.dispatcher.__init__(self)
self.connection_map = {} # connection => connection
self.clients = [] # connections that want messages
self.server_map = ServerMap()
self.trigger = None
self.trigger_magic = None # must be matched to accept trigger
# of (client, message, handler)
self.q_in = FifoQueue(queue_in_size)
# of (target, packet)
# where target is a client to send to, or None to send to all
self.q_out = FifoQueue(queue_out_size)
# PST: could possibly use UNIX sockets when not windows
self._init_server(("", port))
self._init_trigger(("127.0.0.1", port))
self.thread = None
# setup basic handlers
def message_rejected(client, message):
print "MessageRejected(Dummy Handler) {%s}" % message
def mapped_name(client, message):
print "MappedName: %s -> %d" % (message.name, message.id)
client.map.add_resolution(message.name, message.id)
self.server_map.set_handler(MessageRejected, message_rejected)
self.server_map.set_handler(MappedName, mapped_name)
def _init_server(self, server_addr):
"""
Setup server socket.
"""
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.bind(server_addr)
self.listen(CONNECTION_QUEUE_SIZE)
self.connection_map[self] = self
def _init_trigger(self, server_addr):
"""
Setup trigger socket.
The trigger is used to interrupt the asyncore loop and force
examination of the outbound message queue. (This is because
``threading'' in Python 2.6 and before is absolutely soupy.)
"""
assert not self.trigger
trigger = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
trigger.connect(server_addr)
trigger.shutdown(socket.SHUT_RD)
# to identify as trigger
self.trigger_magic = "TRIGGER:%08x" % random.randint(0, 0xffffffff)
trigger.send(self.trigger_magic)
self.trigger = trigger
def packet_read(self, conn, raw_packet):
"""
Read in a packet and dispatch -- asyncore callback.
"""
packet = Packet.decode(raw_packet)
print "packet in with protocol"
if Packet.is_named(packet.protocol):
# protocol is named, resolve id first
name = packet.protocol
id = self.server_map.resolve_id(name)
if id:
named = MappedName()
named.name = name
named.id = id
msg = Message(0, named)
self.q_out.fifo_put((conn, msg))
else:
print "unmapped name: " + str(name)
rejected = MessageRejected()
rejected.id = 0
rejected.reason = rejected.UNMAPPED_NAME
rejected.message = "Unmapped name: '%s'" % name
msg = Message(0, rejected)
self.q_out.fifo_put((conn, msg))
# stop processing
return
else:
# just take id
id = packet.protocol
# this seems slightly silly in the case where the message was
# a named message (it shouldn't hurt though)
name = self.server_map.resolve_name(id)
if not name:
# if the reverse name lookup fails
print "unmapped id: " + str(id)
rejected = MessageRejected()
rejected.id = 0
rejected.reason = rejected.UNMAPPED_ID
rejected.message = "Unmapped ID: %d" % id
msg = Message(0, rejected)
self.q_out.fifo_put((conn, msg))
# stop processing
return
protobuf_class = self.server_map.resolve_protobuffer(name)
assert protobuf_class
protobuf = protobuf_class()
protobuf.ParseFromString(packet.payload)
message = Message(packet.track_id, protobuf, name=name)
print "have protobuf: {" + str(protobuf) + "}"
self.q_in.fifo_put((conn, message))
def handle_accept(self):
"""
Process new connections (asyncore callback).
"""
client, addr = self.accept()
holding = Holding(PROTOMAP_MAGIC, self, client, self.trigger_magic)
self.attach_connection(holding)
def attach_connection(self, new_conn):
"""
Attach a new connection.
"""
self.alter_connection(None, new_conn)
def alter_connection(self, old_conn, new_conn):
"""
Alter an existing connection handler; if new_conn is None the
connection is closed. The old_conn, if any, is removed.
"""
if old_conn and new_conn:
# prevent socket-swapping
assert old_conn.socket is new_conn.socket
# remove existing (if any)
if old_conn in self.connection_map:
del self.connection_map[old_conn]
if old_conn in self.clients:
self.clients.remove(old_conn)
# attach new (if any)
if new_conn:
self.connection_map[new_conn] = new_conn
if isinstance(new_conn, ProtoClient):
self.clients.append(new_conn)
elif old_conn:
# no new_conn, but old_conn -- close connection
old_conn.close()
def connection_closed(self, conn):
"""
Remove a connection.
"""
self.log.debug("Connection closed")
if conn in self.clients:
self.clients.remove(conn)
del self.connection_map[conn]
#
# Internal
#
# PST- can avoid this and keep asyncore loop tighter?
# PST- could possibly send packet to closed connection
def _send_queued_packets(self):
"""
Flush all queued out packets onto the wire.
"""
def send(client, message):
protocol = client.map.resolve_id(message.name) or message.name
packet = message.encode(protocol)
print "writing packet " + str(protocol)
client.write_packet(packet)
while not self.q_out.empty():
client, message = self.q_out.get()
if client:
send(client, message)
else:
for client in self.clients:
send(client, message)
def _thread_pump(self):
"""
Pump that runs inside the dispatch thread. This never normally
terminates.
"""
self.log.debug("Pump started")
while True:
asyncore.loop(SELECT_IDLE_SECONDS, False, self.connection_map, 1)
self._send_queued_packets()
#
# Public
#
def process_messages(self):
"""
Invoke handlers for messages in the incoming queue.
This method should be invoked in the thread where the handlers
are to be executed.
"""
while not self.q_in.empty():
client, message = self.q_in.get()
handler = self.server_map.resolve_handler(message.name)
handler(client, message)
def write(self, message, target=None):
"""
Write a message.
The message is sent to target. If target is None the message
is sent to all connected clients. This method is safe to call
outside the event thread.
"""
self.q_out.fifo_put((target, message))
self.trigger.send('t')
def start(self):
"""
Spawn a thread and start running the server.
Returns self.
"""
def launcher():
"Terminate on pump exception"
try:
self._thread_pump()
except:
traceback.print_exc()
print >> sys.stderr, "!!! ProtoServer Terminated !!!"
os._exit(1)
self.thread = thread.start_new_thread(launcher, ())
return self