def test_vehicle_data_logging():
proc = wake_up_maya(start_position=None, constraints_for_training=None)
# read data
socket = Socket(SUB)
socket.set_string_option(SUB, SUB_SUBSCRIBE, b"Publish Counter")
socket.connect(FLAGS.log_port)
# send control commands
socket_out = Socket(PAIR)
socket_out.connect(FLAGS.control_port)
num_of_agents = 9
num_of_steps = 30
for steps in range(num_of_steps):
for ptr in range(num_of_agents):
vehicle_data = socket.recv()
msg = parse_data(vehicle_data)
tf.logging.info("%s" % msg)
control_signal = "%s:%s" % (int(time.time() * 1000), 0)
motor = 0.5
brake = 0.
steer = 0.
hand_brake = 0.
left_light = 0
right_light = 0
control_signal += ":%s:%s:%s:%s:%s:%s:%s" % ("my_ego_vehicle", motor,
brake, steer, hand_brake,
left_light, right_light)
socket_out.send(control_signal.encode())
maya_to_sleep(proc)
def __init__(self, configfile: str = '', gateway: str = "CTP.MD"):
super(RecorderEngine, self).__init__(event_engine=EventEngine(10))
"""
two sockets to send and recv msg
"""
self.__active = False
self._thread = Thread(target=self._run)
self.id = os.getpid()
self.engine_type = EngineType.LIVE
self._recv_sock = Socket(SUB)
self._send_sock = Socket(PUSH)
if configfile:
self.config_filename = configfile
if gateway:
self.gateway = gateway
filepath = Path.cwd().joinpath("etc/" + self.config_filename)
with open(filepath, encoding='utf8') as fd:
self._config = yaml.load(fd)
self.tick_recordings = {}
self.bar_recordings = {}
self.bar_generators = {}
self.contracts = {}
self.subscribed = False
self.dayswitched = False
self.init_engine()
def __init__(self, event_engine):
self._event_engine = event_engine
self._tick_sock = Socket(SUB)
self._msg_sock = Socket(PAIR)
self._active = False
self._thread = Thread(target=self._run)
def __init__(self, ui_event_engine, outgoing_quue):
self._ui_event_engine = ui_event_engine
self._outgoing_quue = outgoing_quue
self._tick_sock = Socket(SUB)
self._msg_sock = Socket(PAIR)
self._active = False
self._thread = Thread(target=self._run)
def __init__(self, configfile: str = '', id: int = 1):
super(StrategyEngine, self).__init__(event_engine=EventEngine(10))
"""
two sockets to send and recv msg
"""
self.__active = False
self.id = os.getpid()
self.engine_type = EngineType.LIVE
self._recv_sock = Socket(SUB)
self._send_sock = Socket(PUSH)
self._handlers = defaultdict(list)
if configfile:
self.config_filename = configfile
filepath = Path.cwd().joinpath("etc/" + self.config_filename)
with open(filepath, encoding='utf8') as fd:
self._config = yaml.load(fd)
self.ordercount = 0
# stragegy manage
self.strategy_setting = {} # strategy_name: dict
self.strategy_data = {} # strategy_name: dict
self.classes = {} # class_name: stategy_class
self.strategies = {} # strategy_name: strategy
# self.classes_id = {} # class_id : strategy
# self.strategies_id = {} # strategy_ID: strategy
self.symbol_strategy_map = defaultdict(
list) # full_symbol: strategy list
self.orderid_strategy_map = {} # vt_orderid: strategy
self.strategy_orderid_map = defaultdict(
set) # strategy_name: client_order_id list
self.stop_order_count = 0 # for generating stop_orderid
self.stop_orders = {} # stop_orderid: stop_order
self.init_thread = None
self.init_queue = Queue()
# order,tick,position ,etc manage
self.ticks = {}
self.orders = {} # clientorder id list
self.trades = {}
self.positions = {}
self.accounts = {}
self.contracts = {}
self.active_orders = {} # SQ id list
self.rq_client = None
self.rq_symbols = set()
self.offset_converter = OffsetConverter(self)
self.autoinited = False
self.autostarted = False
self.dayswitched = False
self.init_engine()
def test_poll_timeout(self):
with Socket(PAIR) as s1:
with Socket(PAIR) as s2:
s1.bind(SOCKET_ADDRESS)
s2.connect(SOCKET_ADDRESS)
start_time = time.time()
timeout = .05
r, _ = poll([s1, s2], [], timeout)
end_time = time.time()
self.assertTrue(end_time-start_time-timeout < .010)
self.assertEqual(0, len(r), "No sockets to read")
def test_read_poll(self):
with Socket(PAIR) as s1:
with Socket(PAIR) as s2:
s1.bind(SOCKET_ADDRESS)
s2.connect(SOCKET_ADDRESS)
r, _ = poll([s1, s2], [], 0)
self.assertEqual(0, len(r), "Precondition nothing to read")
sent = b'ABC'
s2.send(sent)
r, _ = poll([s1, s2], [], 0)
self.assertTrue(s1 in r, "Socket is in read list")
received = s1.recv()
def __init__(self,config:dict):
super(TradeEngine, self).__init__()
"""
two sockets to send and recv msg
"""
self.__active = False
self.id = 0
self.engine_type = EngineType.LIVE
self._recv_sock = Socket(SUB)
self._send_sock = Socket(PUSH)
self._config = config
self._handlers = defaultdict(list)
# stragegy manage
self.strategy_setting = {} # strategy_name: dict
self.strategy_data = {} # strategy_name: dict
self.classes = {} # class_name: stategy_class
self.strategies = {} # strategy_name: strategy
self.symbol_strategy_map = defaultdict(
list) # vt_symbol: strategy list
self.orderid_strategy_map = {} # vt_orderid: strategy
self.strategy_orderid_map = defaultdict(
set) # strategy_name: orderid list
self.stop_order_count = 0 # for generating stop_orderid
self.stop_orders = {} # stop_orderid: stop_order
self.init_thread = None
self.init_queue = Queue()
# order,tick,position ,etc manage
self.ticks = {}
self.orders = {}
self.trades = {}
self.positions = {}
self.accounts = {}
self.contracts = {}
self.active_orders = {}
self.rq_client = None
self.rq_symbols = set()
self.offset_converter = OffsetConverter(self)
self.init_engine()
def test_subscribe_works(self):
with Socket(PUB) as s1:
with Socket(SUB) as s2:
s1.bind(SOCKET_ADDRESS)
s2.connect(SOCKET_ADDRESS)
s2.set_string_option(SUB, SUB_SUBSCRIBE, 'test')
s1.send('test')
s2.recv()
s1.send('a') # should not get received
expected = b'test121212'
s1.send(expected)
actual = s2.recv()
self.assertEquals(expected, actual)
def iogate(type, queuein=Queue.Queue(), queueout=Queue.Queue(), network={}):
sock = Socket(network['type'])
if type == 'in':
for port in network['ports']:
sock.bind('tcp://0.0.0.0:{port}'.format(port=port))
elif type == 'out':
for node in network['nodes']:
sock.connect('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
def receive_messages():
while True:
queuein.put({'socket': in_sock, 'data': in_sock.recv()})
print("Message received")
def send_messages():
while True:
sock.send(queueout.get(block=True))
print("Message has been sent")
receiver = threading.Thread(target=receive_messages)
sender = threading.Thread(target=send_messages)
receiver.start()
sender.start()
return (queuein, queueout)
def _init_socket():
'''
Initializes nanomsg socket
'''
global _socket
logger.info("Initializing nanomsg socket for address '%s'...",
params.IPC_ADDRESS)
# Close socket if necessary
try:
if _socket:
_socket.close()
except Exception as e:
logger.error("Failed to close nanomsg socket (%s)", e)
return
# Initialize
try:
_socket = Socket(PAIR)
_socket.bind(params.IPC_ADDRESS)
_socket.send_timeout = params.TIMEOUT
except Exception as e:
logger.error("Failed to init nanomsg socket for address '%s' (%s)",
params.IPC_ADDRESS, e)
return
def connect(self):
"""
Connect to kudu
"""
self._socket = Socket(PUSH)
self._socket.connect(self._address)
def server(url):
sock = Socket(PUB)
sock.bind(url)
channel1 = b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
channel2 = b'\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
channel3 = b'\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
channel4 = b'\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
doublearray = array.array('d')
for i in range(65536):
doublearray.insert(i, 0.000001 + i)
doublebytes = doublearray.tobytes()
# databytes = doublearray.tobytes()
# pipes1 = channel1.decode('utf-8')
# pipes2 = channel2.decode('utf-8')
# pipes3 = channel3.decode('utf-8')
# pipes4 = channel4.decode('utf-8')
# datas = databytes.decode('utf-8')
# pipedatas1 = pipes1+datas
# pepedata1 = pipedatas1.encode('utf-8')
# pipedatas2 = pipes2+datas
# pepedata2 = pipedatas2.encode('utf-8')
# pipedatas3 = pipes3+datas
# pepedata3 = pipedatas3.encode('utf-8')
# pipedatas4 = pipes4+datas
# pepedata4 = pipedatas4.encode('utf-8')
while True:
sock.send(b''.join([channel1, doublebytes]))
sock.send(b''.join([channel2, doublebytes]))
sock.send(b''.join([channel3, doublebytes]))
sock.send(b''.join([channel4, doublebytes]))
def start(self, timer=True):
"""
start the mq thread
"""
self._recv_sock = Socket(SUB)
self._send_sock = Socket(PUSH)
self._monitor_sock = Socket(SUB)
# print(os.getpid())
self._recv_sock.connect(self._config['serverpub_url'])
self._recv_sock.set_string_option(SUB, SUB_SUBSCRIBE, '')
self._recv_sock.set_int_option(SOL_SOCKET, RCVTIMEO, 100)
self._send_sock.connect(self._config['serverpull_url'])
self._monitor_sock.connect(self._config['serversub_url'])
self._active = True
if not self._thread.isAlive():
self._thread.start()
def __init__(self, server_addr, direction):
super().__init__(direction + '_stream')
self.server_addr = 'tcp://{}:8081'.format(server_addr)
self.sock = Socket(SUB)
self.sock.connect(self.server_addr)
self.sock.set_string_option(SUB, SUB_SUBSCRIBE, '')
print('Started streaming capture source {} from {}'.format(
self.direction, self.server_addr))
def _bg(self):
sock = Socket(REQ)
sock.connect('inproc://server')
while self.running:
for func in self.bg_tasks:
msg = self.dump((func, []))
sock.send(msg)
self.load(sock.recv())
time.sleep(10)
def test_image_logging():
proc = wake_up_maya(start_position=None, constraints_for_training=None)
socket = Socket(SUB)
socket.set_string_option(SUB, SUB_SUBSCRIBE, b"image")
socket.connect(FLAGS.log_port_image)
image = socket.recv()
ts, image = parse_image(image)
tf.logging.info("ts: %s image shape: %s" % (ts, image.shape))
maya_to_sleep(proc)
def test_lidar_logging():
proc = wake_up_maya(start_position=None, constraints_for_training=None)
socket = Socket(SUB)
socket.set_string_option(SUB, SUB_SUBSCRIBE, b"lidar")
socket.connect(FLAGS.log_port_lidar)
lidar = socket.recv()
ts, point_cloud = parse_lidar_data(lidar)
tf.logging.info("timestamp: %s, point cloud: %s" % (ts, point_cloud))
maya_to_sleep(proc)
def brain_loop(args):
sock_node = Socket(BUS)
sock_node.bind('tcp://*:%s' % BRAIN_PORT)
sock_node.recv_timeout = 250
#sock_node.send_buffer_size = 1000
sock_node.send_timeout = 200
seq = 0
timer = set_mode(args.mode)
server = Server(args)
while True:
ch, value = chanselect([timer.chan, server.chan], [])
#print 'AFTER chanselect', ch is timer.chan, time.time()
if ch is timer.chan:
lights.only(*value)
#print "LIGHTS", value
try:
seq += 1
sock_node.send(
'%i %s %s' %
(seq, CMD_LIGHTS, ' '.join(lights.rev_lookup[led_pin]
for led_pin in value)))
except Exception as ex:
#print ex
pass
elif ch is server.chan:
if value[0] == 'status':
node = None
try:
seq += 1
sock_node.send('%i PING' % seq)
while True:
node_msg = sock_node.recv().split()
if int(node_msg[0]) == seq:
node = 'ok'
break
elif int(node_msg[0]) > seq:
raise Exception('Skipped ping message')
except Exception as ex:
node = repr(ex)
value[1].put({'loop': 'ok', 'node': node})
elif value[0] == 'set_period':
timer.period = value[1]
elif value[0] == 'trip':
timer.trip = True
elif value[0] == 'mode':
mode = value[1]
print("new mode tho %s" % mode)
timer = set_mode(mode)
else:
print "UNKNOWN COMMAND:", value
time.sleep(2)
def test_hd_map_logging():
proc = wake_up_maya(start_position=None, constraints_for_training=None)
socket = Socket(SUB)
socket.set_string_option(SUB, SUB_SUBSCRIBE, b"hd_map")
socket.connect(FLAGS.log_port_hd_map)
hd_map = socket.recv()
my_hd_map = parse_hd_map(hd_map)
tf.logging.info("%s" % my_hd_map)
maya_to_sleep(proc)
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgPublisher, self).__init__(topic)
self._socket = Socket(PUB)
self._socket.send_timeout = 1 # Wait 1ms for sending
self._socket.bind(url)
self._logger = logging.getLogger('NanomsgPublisher')
def connect():
sock = Socket(PAIR)
try:
sock.connect(IPC_ADDRESS)
sock.send_timeout = 10000
sock.recv_timeout = 10000
except Exception as e:
logger.error(
"Failed to connect to address '%s' (%s)", IPC_ADDRESS, e)
sock.close()
raise SystemExit(1)
return sock
def __init__(self, url, event):
""" Initializes object
Args:
url (String): Url to bind to
"""
super(PullWorker, self).__init__()
self._socket = Socket(PULL)
self._url = url
self._socket.bind(self._url)
self._queue = Queue.Queue()
self._stop = event
def run(self):
with Socket(PUB) as sock:
sock.bind(SERVER_ADDR)
while True:
res = self.capture_source_framework.get_next_frame()
if res in [
camera_message_framework.FRAMEWORK_QUIT,
camera_message_framework.FRAMEWORK_DELETED
]:
continue
img, _ = res
sock.send(pack_image(img))
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgSubscriber, self).__init__(topic)
self._socket = Socket(SUB)
self._socket.recv_timeout = 500 # Wait 500ms for receiving msgs
self._socket.connect(url)
self._socket.set_string_option(SUB, SUB_SUBSCRIBE, topic + '|')
self._logger = logging.getLogger('NanomsgSubscriber')
def test_write_poll(self):
with Socket(PAIR) as s1:
with Socket(PAIR) as s2:
s1.bind(SOCKET_ADDRESS)
s2.connect(SOCKET_ADDRESS)
r, w = poll([], [s1, s2], 2)
self.assertEqual(2, len(w), "Precondition nothing to read")
sent = b'ABCD'
# minimum send size
snd_buf_size = 128*1024
s2.set_int_option(SOL_SOCKET, SNDBUF, snd_buf_size)
# send until full
for i in range(snd_buf_size//len(sent)+1):
try:
s2.send(sent, DONTWAIT)
except:
pass
# poll
r, w = poll([], [s1, s2], 0)
self.assertTrue(s2 not in w, "Socket is in write list")
received = s1.recv()
def receiver(queue, ports, stype=SUB):
in_sock = Socket(stype)
for port in ports:
in_sock.bind('tcp://0.0.0.0:{port}'.format(port=port))
def receive_messages():
while True:
queue.put(in_sock.recv())
print("Message received")
receiver = threading.Thread(target=receive_messages)
receiver.start()
return in_sock
def sender(queue, network, stype=PUSH):
out_sock = Socket(stype)
for node in network['nodes']:
print('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
out_sock.connect('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
def send_messages():
while True:
out_sock.send(queue.get(block=True))
print("Message has been sent")
receiver = threading.Thread(target=send_messages)
receiver.start()
return out_sock
def node_loop(brain):
sock_brain = Socket(BUS)
sock_brain.connect('tcp://%s:%s' % (brain, BRAIN_PORT))
while True:
msg = sock_brain.recv()
#print 'HEARD', msg
bits = msg.split()
msgid = bits[0]
cmd = bits[1]
if cmd == CMD_PING:
sock_brain.send('%s PONG' % msgid)
elif cmd == CMD_LIGHTS:
which_pins = [lights.lookup[lgt] for lgt in bits[2:]]
lights.only(*which_pins)
else:
print 'Unhandleable message: %r' % msg
def __init__(self, state, logs, worker_class):
self.state = state
self.logs = logs
self.worker_class = worker_class
self.sock = Socket(REP)
self.sock.bind('inproc://server')
self.sock.bind('ipc://socket')
self.results = {}
self.threads = {}
self.running = True
self.change_lock = Lock()
self.id_gen = iter(range(10000000)).next
self.init()
