def iogate(type, queuein=Queue.Queue(), queueout=Queue.Queue(), network={}):
sock = Socket(network['type'])
if type == 'in':
for port in network['ports']:
sock.bind('tcp://0.0.0.0:{port}'.format(port=port))
elif type == 'out':
for node in network['nodes']:
sock.connect('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
def receive_messages():
while True:
queuein.put({'socket':in_sock, 'data': in_sock.recv()})
print("Message received")
def send_messages():
while True:
sock.send(queueout.get(block=True))
print("Message has been sent")
receiver = threading.Thread(target=receive_messages)
sender = threading.Thread(target=send_messages)
receiver.start()
sender.start()
return (queuein, queueout)
def __init__(self, configfile: str = '', gateway: str = "CTP.MD"):
super(RecorderEngine, self).__init__(event_engine=EventEngine(10))
"""
two sockets to send and recv msg
"""
self.__active = False
self._thread = Thread(target=self._run)
self.id = os.getpid()
self.engine_type = EngineType.LIVE
self._recv_sock = Socket(SUB)
self._send_sock = Socket(PUSH)
if configfile:
self.config_filename = configfile
if gateway:
self.gateway = gateway
filepath = Path.cwd().joinpath("etc/" + self.config_filename)
with open(filepath, encoding='utf8') as fd:
self._config = yaml.load(fd)
self.tick_recordings = {}
self.bar_recordings = {}
self.bar_generators = {}
self.contracts = {}
self.subscribed = False
self.dayswitched = False
self.init_engine()
def __init__(self, event_engine):
self._event_engine = event_engine
self._tick_sock = Socket(SUB)
self._msg_sock = Socket(PAIR)
self._active = False
self._thread = Thread(target=self._run)
def _init_socket():
'''
Initializes nanomsg socket
'''
global _socket
logger.info("Initializing nanomsg socket for address '%s'...",
params.IPC_ADDRESS)
# Close socket if necessary
try:
if _socket:
_socket.close()
except Exception as e:
logger.error("Failed to close nanomsg socket (%s)", e)
return
# Initialize
try:
_socket = Socket(PAIR)
_socket.bind(params.IPC_ADDRESS)
_socket.send_timeout = params.TIMEOUT
except Exception as e:
logger.error("Failed to init nanomsg socket for address '%s' (%s)",
params.IPC_ADDRESS, e)
return
def connect(self):
"""
Connect to kudu
"""
self._socket = Socket(PUSH)
self._socket.connect(self._address)
def __init__(self, ui_event_engine, outgoing_quue):
self._ui_event_engine = ui_event_engine
self._outgoing_quue = outgoing_quue
self._tick_sock = Socket(SUB)
self._msg_sock = Socket(PAIR)
self._active = False
self._thread = Thread(target=self._run)
def __init__(self, server_addr, direction):
super().__init__(direction + '_stream')
self.server_addr = 'tcp://{}:8081'.format(server_addr)
self.sock = Socket(SUB)
self.sock.connect(self.server_addr)
self.sock.set_string_option(SUB, SUB_SUBSCRIBE, '')
print('Started streaming capture source {} from {}'.format(
self.direction, self.server_addr))
def __init__(self, configfile: str = '', id: int = 1):
super(StrategyEngine, self).__init__(event_engine=EventEngine(10))
"""
two sockets to send and recv msg
"""
self.__active = False
self.id = os.getpid()
self.engine_type = EngineType.LIVE
self._recv_sock = Socket(SUB)
self._send_sock = Socket(PUSH)
self._handlers = defaultdict(list)
if configfile:
self.config_filename = configfile
filepath = Path.cwd().joinpath("etc/" + self.config_filename)
with open(filepath, encoding='utf8') as fd:
self._config = yaml.load(fd)
self.ordercount = 0
# stragegy manage
self.strategy_setting = {} # strategy_name: dict
self.strategy_data = {} # strategy_name: dict
self.classes = {} # class_name: stategy_class
self.strategies = {} # strategy_name: strategy
# self.classes_id = {} # class_id : strategy
# self.strategies_id = {} # strategy_ID: strategy
self.symbol_strategy_map = defaultdict(
list) # full_symbol: strategy list
self.orderid_strategy_map = {} # vt_orderid: strategy
self.strategy_orderid_map = defaultdict(
set) # strategy_name: client_order_id list
self.stop_order_count = 0 # for generating stop_orderid
self.stop_orders = {} # stop_orderid: stop_order
self.init_thread = None
self.init_queue = Queue()
# order,tick,position ,etc manage
self.ticks = {}
self.orders = {} # clientorder id list
self.trades = {}
self.positions = {}
self.accounts = {}
self.contracts = {}
self.active_orders = {} # SQ id list
self.rq_client = None
self.rq_symbols = set()
self.offset_converter = OffsetConverter(self)
self.autoinited = False
self.autostarted = False
self.dayswitched = False
self.init_engine()
def main():
ap = argparse.ArgumentParser()
ap.add_argument('-n', '--requests', metavar="NUM",
help="Number of requests to issue (default %(default)d)",
default=10000, type=int)
ap.add_argument('-c', '--concurrent', metavar="NUM",
help="Number of requests sent simultaneously (default %(default)d)",
default=1000, type=int)
ap.add_argument('-m', '--max-value', metavar="NUM",
help="Maximum number that's sent for factorizing (default %(default)d)",
default=10**12, type=int)
ap.add_argument('--min-value', metavar="NUM",
help="Maximum number that's sent for factorizing (default %(default)d)",
default=10**11, type=int)
ap.add_argument('--topology', metavar="URL", required=True,
help="Url for topology to join to")
options = ap.parse_args()
sock = Socket(REQ, domain=AF_SP_RAW)
sock.set_string_option(SOL_SOCKET, SOCKET_NAME, "client")
sock.configure(options.topology)
start_time = time.time()
reqiter = requests(options)
req = {}
for i in range(options.concurrent):
rid, val = next(reqiter)
sock.send(b'\x00\x00\x00\x00' + struct.pack('>L', rid | 0x80000000)
+ str(val).encode('ascii'))
req[rid] = val
errors = 0
sp = 0
n = 0
while req:
data = sock.recv()
rid = struct.unpack_from('>L', data)[0] & ~0x80000000
factors = map(int, data[4:].decode('ascii').split(','))
checkval = reduce(int.__mul__, factors)
if rid not in req:
sp += 1
elif req.pop(rid) != checkval:
errors += 1
else:
n += 1
try:
rid, val = next(reqiter)
except StopIteration:
continue
else:
sock.send(b'\x00\x00\x00\x00' + struct.pack('>L', rid | 0x80000000)
+ str(val).encode('ascii'))
req[rid] = val
sec = time.time() - start_time
print("Done", options.requests, "requests in", sec,
"seconds, errors:", errors, ", spurious messages:", sp)
def test_poll_timeout(self):
with Socket(PAIR) as s1:
with Socket(PAIR) as s2:
s1.bind(SOCKET_ADDRESS)
s2.connect(SOCKET_ADDRESS)
start_time = time.time()
timeout = .05
r, _ = poll([s1, s2], [], timeout)
end_time = time.time()
self.assertTrue(end_time-start_time-timeout < .010)
self.assertEqual(0, len(r), "No sockets to read")
def __init__(self, url, event):
""" Initializes object
Args:
url (String): Url to bind to
"""
super(PullWorker, self).__init__()
self._socket = Socket(PULL)
self._url = url
self._socket.bind(self._url)
self._queue = Queue.Queue()
self._stop = event
def test_read_poll(self):
with Socket(PAIR) as s1:
with Socket(PAIR) as s2:
s1.bind(SOCKET_ADDRESS)
s2.connect(SOCKET_ADDRESS)
r, _ = poll([s1, s2], [], 0)
self.assertEqual(0, len(r), "Precondition nothing to read")
sent = b'ABC'
s2.send(sent)
r, _ = poll([s1, s2], [], 0)
self.assertTrue(s1 in r, "Socket is in read list")
received = s1.recv()
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgPublisher, self).__init__(topic)
self._socket = Socket(PUB)
self._socket.send_timeout = 1 # Wait 1ms for sending
self._socket.bind(url)
self._logger = logging.getLogger('NanomsgPublisher')
class StreamClient(CaptureSource):
def __init__(self, server_addr, direction):
super().__init__(direction + '_stream')
self.server_addr = 'tcp://{}:8081'.format(server_addr)
self.sock = Socket(SUB)
self.sock.connect(self.server_addr)
self.sock.set_string_option(SUB, SUB_SUBSCRIBE, '')
print('Started streaming capture source {} from {}'.format(
self.direction, self.server_addr))
def acquire_next_image(self):
img = unpack_image(self.sock.recv())
return img, time.time()
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgSubscriber, self).__init__(topic)
self._socket = Socket(SUB)
self._socket.recv_timeout = 500 # Wait 500ms for receiving msgs
self._socket.connect(url)
self._socket.set_string_option(SUB, SUB_SUBSCRIBE, topic + '|')
self._logger = logging.getLogger('NanomsgSubscriber')
def nano_server(endpoint):
s = Socket(PAIR)
s.bind('ipc://' + endpoint)
#s.bind('tcp://127.0.0.1:9001')
try:
while True:
s.recv()
s.send('ok')
finally:
s.close()
def server(url):
sock = Socket(PUB)
sock.bind(url)
channel1 = b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
channel2 = b'\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
channel3 = b'\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
channel4 = b'\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
doublearray = array.array('d')
for i in range(65536):
doublearray.insert(i, 0.000001 + i)
doublebytes = doublearray.tobytes()
# databytes = doublearray.tobytes()
# pipes1 = channel1.decode('utf-8')
# pipes2 = channel2.decode('utf-8')
# pipes3 = channel3.decode('utf-8')
# pipes4 = channel4.decode('utf-8')
# datas = databytes.decode('utf-8')
# pipedatas1 = pipes1+datas
# pepedata1 = pipedatas1.encode('utf-8')
# pipedatas2 = pipes2+datas
# pepedata2 = pipedatas2.encode('utf-8')
# pipedatas3 = pipes3+datas
# pepedata3 = pipedatas3.encode('utf-8')
# pipedatas4 = pipes4+datas
# pepedata4 = pipedatas4.encode('utf-8')
while True:
sock.send(b''.join([channel1, doublebytes]))
sock.send(b''.join([channel2, doublebytes]))
sock.send(b''.join([channel3, doublebytes]))
sock.send(b''.join([channel4, doublebytes]))
def master_loop():
sock_slave = Socket(PAIR)
sock_slave.bind('tcp://*:%s' % MASTER_PORT)
sock_slave.recv_timeout = 250
#sock_slave.send_buffer_size = 1000
sock_slave.send_timeout = 200
seq = 0
timer = LightTimer()
server = Server()
while True:
ch, value = chanselect([timer.chan, server.chan], [])
#print 'AFTER chanselect', ch is timer.chan, time.time()
if ch is timer.chan:
lights.only(*value)
#print "LIGHTS", value
try:
seq += 1
sock_slave.send('%i %s %s' % (
seq,
CMD_LIGHTS,
' '.join(lights.rev_lookup[led_pin] for led_pin in value)))
except Exception as ex:
#print ex
pass
elif ch is server.chan:
if value[0] == 'status':
slave = None
try:
seq += 1
sock_slave.send('%i PING' % seq)
while True:
slave_msg = sock_slave.recv().split()
if int(slave_msg[0]) == seq:
slave = 'ok'
break
elif int(slave_msg[0]) > seq:
raise Exception('Skipped ping message')
except Exception as ex:
slave = repr(ex)
value[1].put({'loop': 'ok', 'slave': slave})
elif value[0] == 'set_period':
timer.period = value[1]
elif value[0] == 'trip':
timer.trip = True
else:
print "UNKNOWN COMMAND:", value
time.sleep(2)
def __init__(self,config:dict):
super(TradeEngine, self).__init__()
"""
two sockets to send and recv msg
"""
self.__active = False
self.id = 0
self.engine_type = EngineType.LIVE
self._recv_sock = Socket(SUB)
self._send_sock = Socket(PUSH)
self._config = config
self._handlers = defaultdict(list)
# stragegy manage
self.strategy_setting = {} # strategy_name: dict
self.strategy_data = {} # strategy_name: dict
self.classes = {} # class_name: stategy_class
self.strategies = {} # strategy_name: strategy
self.symbol_strategy_map = defaultdict(
list) # vt_symbol: strategy list
self.orderid_strategy_map = {} # vt_orderid: strategy
self.strategy_orderid_map = defaultdict(
set) # strategy_name: orderid list
self.stop_order_count = 0 # for generating stop_orderid
self.stop_orders = {} # stop_orderid: stop_order
self.init_thread = None
self.init_queue = Queue()
# order,tick,position ,etc manage
self.ticks = {}
self.orders = {}
self.trades = {}
self.positions = {}
self.accounts = {}
self.contracts = {}
self.active_orders = {}
self.rq_client = None
self.rq_symbols = set()
self.offset_converter = OffsetConverter(self)
self.init_engine()
def receiver(queue, ports, stype=SUB):
in_sock = Socket(stype)
for port in ports:
in_sock.bind('tcp://0.0.0.0:{port}'.format(port=port))
def receive_messages():
while True:
queue.put(in_sock.recv())
print("Message received")
receiver = threading.Thread(target=receive_messages)
receiver.start()
return in_sock
def test_subscribe_works(self):
with Socket(PUB) as s1:
with Socket(SUB) as s2:
s1.bind(SOCKET_ADDRESS)
s2.connect(SOCKET_ADDRESS)
s2.set_string_option(SUB, SUB_SUBSCRIBE, 'test')
s1.send('test')
s2.recv()
s1.send('a') # should not get received
expected = b'test121212'
s1.send(expected)
actual = s2.recv()
self.assertEquals(expected, actual)
def receiver(queue, ports, stype=SUB):
in_sock = Socket(stype)
for port in ports:
in_sock.bind('tcp://0.0.0.0:{port}'.format(port=port))
def receive_messages():
while True:
queue.put(in_sock.recv())
print("Message received")
receiver = threading.Thread(target=receive_messages)
receiver.start()
return in_sock
def __init__(self, url, recv_timeout=1000, send_timeout=1000):
self.url = url.encode()
self.socket = Socket(REQ)
self.socket.recv_timeout = recv_timeout
self.socket.send_timeout = send_timeout
super(ConsoleProxy, self).__init__()
def main():
ap = argparse.ArgumentParser()
ap.add_argument('-n', '--rate-limit', metavar="NUM",
help="Number of requests to issue per second (default %(default)d)",
default=100, type=float)
ap.add_argument('-m', '--max-value', metavar="NUM",
help="Maximum number that's sent for factorizing (default %(default)d)",
default=10**12, type=int)
ap.add_argument('--min-value', metavar="NUM",
help="Maximum number that's sent for factorizing (default %(default)d)",
default=10**11, type=int)
ap.add_argument('--topology', metavar="URL", required=True,
help="Url for topology to join to")
options = ap.parse_args()
delay = 1.0 / options.rate_limit
sock = Socket(PUSH)
sock.set_string_option(SOL_SOCKET, SOCKET_NAME, "push")
sock.configure(options.topology)
while True:
tm = time.time()
num = random.randint(options.min_value, options.max_value)
sock.send(str(num))
to_sleep = tm + delay - time.time()
if to_sleep > 0:
time.sleep(to_sleep)
def __init__(self, state, logs, worker_class):
self.state = state
self.logs = logs
self.worker_class = worker_class
self.sock = Socket(REP)
self.sock.bind('inproc://server')
self.sock.bind('ipc://socket')
self.results = {}
self.threads = {}
self.running = True
self.change_lock = Lock()
self.id_gen = iter(range(10000000)).next
self.init()
def sender(queue, network, stype=PUSH):
out_sock = Socket(stype)
for node in network['nodes']:
print('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
out_sock.connect('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
def send_messages():
while True:
out_sock.send(queue.get(block=True))
print("Message has been sent")
receiver = threading.Thread(target=send_messages)
receiver.start()
return out_sock
def sender(queue, addresses, stype):
"""
Bind a queue to a connecting nanomsg socket's multiple endpoints
in a separate thread.
Parameters
----------
queue : Queue
A Queue object to be emptied by sender socket
addresses : list
A list of strings of format '<protocol>://<ip>:<port>' to bind to
stype : int
One of the nanomsg scalability socket types: PUSH, PULL, PUB, SUB, PAIR
Returns
-------
nanomsg socket object
"""
with err.applicationbound():
out_sock = Socket(stype)
out_sock.set_int_option(SOL_SOCKET, SNDTIMEO, 1000)
for address in addresses:
endpoint = out_sock.connect(address)
out_endpoints.append(endpoint)
def send_messages():
""" """
while True:
try:
out_sock.send(queue.get(block=True))
log.info("Message has been sent")
except NanoMsgAPIError as e:
log.debug(e)
log.debug(dir(e))
receiver = threading.Thread(target=send_messages)
receiver.start()
return out_sock
def sender(queue, addresses, stype):
"""
Bind a queue to a connecting nanomsg socket's multiple endpoints
in a separate thread.
Parameters
----------
queue : Queue
A Queue object to be emptied by sender socket
addresses : list
A list of strings of format '<protocol>://<ip>:<port>' to bind to
stype : int
One of the nanomsg scalability socket types: PUSH, PULL, PUB, SUB, PAIR
Returns
-------
nanomsg socket object
"""
with err.applicationbound():
out_sock = Socket(stype)
out_sock.set_int_option(SOL_SOCKET, SNDTIMEO, 1000)
for address in addresses:
endpoint = out_sock.connect(address)
out_endpoints.append(endpoint)
def send_messages():
""" """
while True:
try:
out_sock.send(queue.get(block=True))
log.info("Message has been sent")
except NanoMsgAPIError as e:
log.debug(e)
log.debug(dir(e))
receiver = threading.Thread(target=send_messages)
receiver.start()
return out_sock
def connect():
sock = Socket(PAIR)
try:
sock.connect(IPC_ADDRESS)
sock.send_timeout = 10000
sock.recv_timeout = 10000
except Exception as e:
logger.error(
"Failed to connect to address '%s' (%s)", IPC_ADDRESS, e)
sock.close()
raise SystemExit(1)
return sock
def _bg(self):
sock = Socket(REQ)
sock.connect('inproc://server')
while self.running:
for func in self.bg_tasks:
msg = self.dump((func, []))
sock.send(msg)
self.load(sock.recv())
time.sleep(10)
def run(self):
with Socket(PUB) as sock:
sock.bind(SERVER_ADDR)
while True:
res = self.capture_source_framework.get_next_frame()
if res in [
camera_message_framework.FRAMEWORK_QUIT,
camera_message_framework.FRAMEWORK_DELETED
]:
continue
img, _ = res
sock.send(pack_image(img))
def test_image_logging():
proc = wake_up_maya(start_position=None, constraints_for_training=None)
socket = Socket(SUB)
socket.set_string_option(SUB, SUB_SUBSCRIBE, b"image")
socket.connect(FLAGS.log_port_image)
image = socket.recv()
ts, image = parse_image(image)
tf.logging.info("ts: %s image shape: %s" % (ts, image.shape))
maya_to_sleep(proc)
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgPublisher, self).__init__(topic)
self._socket = Socket(PUB)
self._socket.send_timeout = 1 # Wait 1ms for sending
self._socket.bind(url)
self._logger = logging.getLogger('NanomsgPublisher')
def test_lidar_logging():
proc = wake_up_maya(start_position=None, constraints_for_training=None)
socket = Socket(SUB)
socket.set_string_option(SUB, SUB_SUBSCRIBE, b"lidar")
socket.connect(FLAGS.log_port_lidar)
lidar = socket.recv()
ts, point_cloud = parse_lidar_data(lidar)
tf.logging.info("timestamp: %s, point cloud: %s" % (ts, point_cloud))
maya_to_sleep(proc)
class ServiceDiscovery(object):
def __init__(self, port, deadline=5000):
self.socket = Socket(SURVEYOR)
self.port = port
self.deadline = deadline
self.services = defaultdict(set)
def bind(self):
self.socket.bind('tcp://*:%s' % self.port)
self.socket.set_int_option(SURVEYOR, SURVEYOR_DEADLINE, self.deadline)
def discover(self):
if not self.socket.is_open():
return self.services
self.services = defaultdict(set)
self.socket.send('service query')
while True:
try:
response = self.socket.recv()
except NanoMsgAPIError:
break
service, address = response.split('|')
self.services[service].add(address)
return self.services
def resolve(self, service):
providers = self.services[service]
if not providers:
return None
return random.choice(tuple(providers))
def close(self):
self.socket.close()
class ServiceDiscovery(object):
def __init__(self, port, deadline=5000):
self.socket = Socket(SURVEYOR)
self.port = port
self.deadline = deadline
self.services = defaultdict(set)
def bind(self):
self.socket.bind('tcp://172.30.42.174:%s' % self.port)
self.socket.set_int_option(SURVEYOR, SURVEYOR_DEADLINE, self.deadline)
def discover(self):
if not self.socket.is_open():
return self.services
self.services = defaultdict(set)
self.socket.send('service query')
while True:
try:
response = self.socket.recv()
except NanoMsgAPIError:
break
service, address = response.split('|')
self.services[service].add(address)
return self.services
def resolve(self, service):
providers = self.services[service]
if not providers:
return None
return random.choice(tuple(providers))
def close(self):
self.socket.close()
def iogate(type, queuein=Queue.Queue(), queueout=Queue.Queue(), network={}):
sock = Socket(network['type'])
if type == 'in':
for port in network['ports']:
sock.bind('tcp://0.0.0.0:{port}'.format(port=port))
elif type == 'out':
for node in network['nodes']:
sock.connect('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
def receive_messages():
while True:
queuein.put({'socket': in_sock, 'data': in_sock.recv()})
print("Message received")
def send_messages():
while True:
sock.send(queueout.get(block=True))
print("Message has been sent")
receiver = threading.Thread(target=receive_messages)
sender = threading.Thread(target=send_messages)
receiver.start()
sender.start()
return (queuein, queueout)
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgSubscriber, self).__init__(topic)
self._socket = Socket(SUB)
self._socket.recv_timeout = 500 # Wait 500ms for receiving msgs
self._socket.connect(url)
self._socket.set_string_option(SUB, SUB_SUBSCRIBE, topic + '|')
self._logger = logging.getLogger('NanomsgSubscriber')
def test_hd_map_logging():
proc = wake_up_maya(start_position=None, constraints_for_training=None)
socket = Socket(SUB)
socket.set_string_option(SUB, SUB_SUBSCRIBE, b"hd_map")
socket.connect(FLAGS.log_port_hd_map)
hd_map = socket.recv()
my_hd_map = parse_hd_map(hd_map)
tf.logging.info("%s" % my_hd_map)
maya_to_sleep(proc)
def sender(queue, network, stype=PUSH):
out_sock = Socket(stype)
out_sock.set_int_option(SOL_SOCKET, SNDTIMEO, 1000)
for node in network['nodes']:
endpoint = out_sock.connect('tcp://{ip}:{port}'.format(
ip=node['ip'], port=node['port']))
out_endpoints.append(endpoint)
def send_messages():
while True:
try:
out_sock.send(queue.get(block=True))
print("Message has been sent")
except NanoMsgAPIError as e:
print(e)
print(dir(e))
receiver = threading.Thread(target=send_messages)
receiver.start()
return out_sock
def nano_sub(bind, tables):
"""Nanomsg fanout subscriber. (Experimental)
This subscriber will use nanomsg to publish the event to outside.
"""
logger = logging.getLogger("meepo.sub.nano_sub")
from nanomsg import Socket, PUB
pub_socket = Socket(PUB)
pub_socket.bind(bind)
def _sub(table):
for action in ("write", "update", "delete"):
def _sub(pk, action=action):
msg = bytes("%s_%s %s" % (table, action, pk), 'utf-8')
logger.debug("pub msg %s" % msg)
pub_socket.send(msg)
signal("%s_%s" % (table, action)).connect(_sub, weak=False)
for table in set(tables):
_sub(table)
def nano_sub(bind, tables):
"""Nanomsg fanout subscriber. (Experimental)
This subscriber will use nanomsg to publish the event to outside.
"""
logger = logging.getLogger("meepo.sub.nano_sub")
from nanomsg import Socket, PUB
pub_socket = Socket(PUB)
pub_socket.bind(bind)
def _sub(table):
for action in ("write", "update", "delete"):
def _sub(pk, action=action):
msg = bytes("%s_%s %s" % (table, action, pk), 'utf-8')
logger.debug("pub msg %s" % msg)
pub_socket.send(msg)
signal("%s_%s" % (table, action)).connect(_sub, weak=False)
for table in set(tables):
_sub(table)
def _bg(self):
sock = Socket(REQ)
sock.connect("inproc://server")
while self.running:
for func in self.bg_tasks:
msg = self.dump((func, []))
sock.send(msg)
self.load(sock.recv())
time.sleep(10)
def receiver(queue, addresses, stype):
"""
Bind a queue to a listening nanomsg socket's multiple endpoints
in a separate thread.
Parameters
----------
queue : Queue
A Queue object to be filled by receiver socket
addresses : list
A list of strings of format '<protocol>://<ip>:<port>' to bind to
stype : int
One of the nanomsg scalability socket types: PUSH, PULL, PUB, SUB, PAIR
Returns
-------
nanomsg socket object
"""
with err.applicationbound():
in_sock = Socket(stype)
for address in addresses:
in_sock.bind(address)
def receive_messages():
""" """
while True:
queue.put(in_sock.recv())
log.info("Message received")
receiver = threading.Thread(target=receive_messages)
receiver.start()
return in_sock
def sender(queue, network, stype=PUSH):
out_sock = Socket(stype)
out_sock.set_int_option(SOL_SOCKET, SNDTIMEO, 1000)
for node in network['nodes']:
endpoint = out_sock.connect('tcp://{ip}:{port}'.format(ip=node['ip'],
port=node['port']))
out_endpoints.append(endpoint)
def send_messages():
while True:
try:
out_sock.send(queue.get(block=True))
print("Message has been sent")
except NanoMsgAPIError as e:
print(e)
print(dir(e))
receiver = threading.Thread(target=send_messages)
receiver.start()
return out_sock
class LogSub(QThread):
def __init__(self, url):
self.socket = Socket(SUB)
self.url = url
self.handlers = []
super().__init__()
def register_handler(self, handler):
self.handlers.append(handler)
def run(self):
self.socket.set_string_option(SUB, SUB_SUBSCRIBE, b'')
self.socket.connect(self.url)
while True:
try:
msg = self.socket.recv()
msg_pack = msgpack.unpackb(msg, encoding='utf-8')
for h in self.handlers:
h(**msg_pack)
except nanomsg.NanoMsgAPIError as e:
raise
def start_service(service_name, service_protocol, service_port):
socket = Socket(REP)
socket.bind('%s://*:%s' % (service_protocol, service_port))
print 'Starting service %s' % service_name
while True:
request = socket.recv()
print 'Request: %s' % request
socket.send('The answer is 42')
def register_service(service_name, service_address, discovery_host,
discovery_port):
socket = Socket(RESPONDENT)
socket.connect('tcp://%s:%s' % (discovery_host, discovery_port))
print 'Starting service registration [service: %s %s, discovery: %s:%s]' \
% (service_name, service_address, discovery_host, discovery_port)
while True:
message = socket.recv()
if message == 'service query':
socket.send('%s|%s' % (service_name, service_address))
def __init__(self, state, logs, worker_class):
self.state = state
self.logs = logs
self.worker_class = worker_class
self.sock = Socket(REP)
self.sock.bind("inproc://server")
self.sock.bind("ipc://socket")
self.results = {}
self.threads = {}
self.running = True
self.change_lock = Lock()
self.id_gen = iter(range(10000000)).next
self.init()
def nano_client(endpoint):
s = Socket(PAIR)
s.connect('ipc://' + endpoint)
#s.connect('tcp://127.0.0.1:9001')
count = 0
payload = test_str(SIZE)
start = time.time()
try:
while True:
s.send(payload)
s.recv()
count += 1
except:
pass
finally:
end = time.time()
total = end - start
print 'total: ', count
print 'took: ', total
print 'req / sec:', count / total
print 'bandwidth: %f MBps' % (((count / total) * SIZE) / 2 ** 20)
s.close()
class NanomsgSubscriber(HiddenSubscriber):
""" Subscriber class subscribing to a certain topic
Attributes:
context (zmq.Context):
socket (Socket): Socket object of ZMQ context
topic (String): Topic subscriber subscribes to
"""
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgSubscriber, self).__init__(topic)
self._socket = Socket(SUB)
self._socket.recv_timeout = 500 # Wait 500ms for receiving msgs
self._socket.connect(url)
self._socket.set_string_option(SUB, SUB_SUBSCRIBE, topic + '|')
self._logger = logging.getLogger('NanomsgSubscriber')
def receive(self):
""" Receives a message
Returns:
String
"""
message = self._socket.recv()
return message[len(self.topic) + 1:]
def __enter__(self):
""" Statement used for the `` with ... as ...:`` returns
the object to use in the ``with`` block
Returns:
NanomsgSubscriber
"""
return self
def __exit__(self, exc_type, exc_value, traceback):
""" Executed when leaving ``with`` block, regardless whether
because of an exception or normal program flow
"""
self._socket.close()
class ConsoleProxy(object):
def __init__(self, url, recv_timeout=1000, send_timeout=1000):
self.url = url.encode()
self.socket = Socket(REQ)
self.socket.recv_timeout = recv_timeout
self.socket.send_timeout = send_timeout
super(ConsoleProxy, self).__init__()
def connect(self):
self.socket.connect(self.url)
def disconnect(self):
self.socket.close()
def send_command(self, cmd_name, echo=True, **kwargs):
command = {
"name": cmd_name,
"args": kwargs
}
dump = msgpack.packb(command)
if echo:
print('send: ', command)
try:
self.socket.send(dump)
except NanoMsgAPIError as e:
print('send_error', e.msg)
pass
try:
recv = self.socket.recv()
unpack_msg = msgpack.unpackb(recv, encoding='utf-8')
if echo:
print('recv: ', recv)
print(unpack_msg)
return unpack_msg
except NanoMsgAPIError as e:
print('recv_error', e.msg)
class Client(Common):
def __init__(self, uri="inproc://server", id=None):
self.sock = Socket(REQ)
self.sock.connect(uri)
self.id = id
def close(self):
self.sock.close()
def call(self, func, *args):
msg = self.dump((func, args))
self.sock.send(msg)
return self.load(self.sock.recv())
def sync_call(self, func, *args):
id = self.call(func, *args)
print "got id", id
while True:
res = self.getresult(id)
if res:
return res
time.sleep(0.1)
def result(self, id, result):
return self.call("result", id, result)
def getresult(self, id):
return self.call("getresult", id)
def getstate(self, key):
return self.call("getstate", key)
def setstate(self, key, value):
return self.call("setstate", key, value)
def getlog(self, id, since=0):
return self.call("getlog", id, since)
def out(self, msg):
return self.call("out", self.id, msg)
def err(self, msg):
return self.call("err", self.id, msg)
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--topology', metavar="URL", required=True,
help="Url for topology to join to")
options = ap.parse_args()
sock = Socket(PULL)
sock.set_string_option(SOL_SOCKET, SOCKET_NAME, "push")
sock.configure(options.topology)
while True:
data = sock.recv()
num, factors = data.decode('ascii').split('=', 1)
factors = map(int, factors.split('*'))
checkval = reduce(int.__mul__, factors)
assert int(num) == checkval
def __init__(self, mouse, gltext, conf):
self.conf = conf
self.mouse = mouse # TODO: use a special mouse object instead of the editor_main object directly.
self.gltext = gltext
self.nodes = []
self.links = []
self.nodes_dict = {} # integers. 16-bit node addresses
self.links_dict = {} # a pair of node objects. (node1, node2) is equivalent to (node2, node1), but only one pair exists in links_dict
self.mouse_hover = False
self.mouse_dragging = False
self.selected = None
self.selected_pos_ofs = vector.Vector()
h = 0. # 10 cm from the ground. nnope. for now, h has to be 0.
r = 1.
for i in range(10):
a = float(i) / (r + 10) * 15.
x, y = r * math.sin(a), r * math.cos(a)
r += 0.5
n = Node( vector.Vector((x, h, y)), i + 1, self._get_node_color(i+1), gltext )
self.append_node(n)
# n = Node( vector.Vector((0., h, 0.)), 1, gltext ); self.append_node(n)
# n = Node( vector.Vector((1., h, 0.)), 2, gltext ); self.append_node(n)
# n = Node( vector.Vector((2., h, -1.)), 3, gltext ); self.append_node(n)
# n = Node( vector.Vector((-2., h, 2.)), 4, gltext ); self.append_node(n)
# n = Node( vector.Vector((-1., h, 2.)), 5, gltext ); self.append_node(n)
# n = Node( vector.Vector((-2., h, 1.)), 6, gltext ); self.append_node(n)
# n = Node( vector.Vector((-1., h, 0.)), 7, gltext ); self.append_node(n)
# n = Node( vector.Vector((-1.5, h, -1.)), 8, gltext ); self.append_node(n)
# n = Node( vector.Vector((0.5, h, 1.)), 9, gltext ); self.append_node(n)
# n = Node( vector.Vector((-1.4, h, 0.5)), 10, gltext ); self.append_node(n)
#n.attrs["etx"] = 44
self.s1 = Socket(SUB)
self.s1.connect('tcp://127.0.0.1:55555')
self.s1.set_string_option(SUB, SUB_SUBSCRIBE, '')
def slave_loop(master):
sock_master = Socket(PAIR)
sock_master.connect('tcp://%s:%s' % (master, MASTER_PORT))
while True:
msg = sock_master.recv()
#print 'HEARD', msg
bits = msg.split()
msgid = bits[0]
cmd = bits[1]
if cmd == CMD_PING:
sock_master.send('%s PONG' % msgid)
elif cmd == CMD_LIGHTS:
which_pins = [lights.lookup[lgt] for lgt in bits[2:]]
lights.only(*which_pins)
else:
print 'Unhandleable message: %r' % msg
def __init__(self, nugui, mouse, gltext, conf):
self.conf = conf
self.mouse = mouse # TODO: use a special mouse object instead of the editor_main object directly.
self.gltext = gltext
self.nugui = nugui
self.world = world.World("ff", self.conf)
self.underworld = world.World("", self.conf) # not visible. used serializing keyframes
self.mouse_x = 0.
self.mouse_y = 0.
self.mouse_hover = False
self.mouse_dragging = False
self.selected = None
self.selected_pos_ofs = vector.Vector()
self.node_renderer = renderers.NodeRenderer(self.gltext)
self.link_renderer = renderers.LinkRenderer()
# saved when closing the windows. loaded at startup.
self.session_filename = os.path.normpath(os.path.join(self.conf.path_database, "session_conf.txt"))
self.load_session()
self.recording = True
self.state = self.STATE_PLAYBACK
self.worldstreamer = world_streamer.WorldStreamer(sync_window_seconds=self.conf.sync_depth_seconds)
#self.current_playback_time = 0. # timepoint of the simulation that is currently visible on screen. can be dragged around with a slider.
self.timeslider_end_time = 0.
self.graph_window = graph_window.GraphWindow(self.gltext)
self.graph_window_initialized = False
self.s1 = Socket(SUB)
self.s1.connect('tcp://127.0.0.1:55555')
self.s1.set_string_option(SUB, SUB_SUBSCRIBE, '')
class NanomsgPublisher(HiddenPublisher):
""" Publisher class publishing messages to a certain topic to an url
Attributes:
context (zmq.Context):
socket (Socket): Socket object of ZMQ context
topic (String): Topic publisher publishs to
"""
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgPublisher, self).__init__(topic)
self._socket = Socket(PUB)
self._socket.send_timeout = 1 # Wait 1ms for sending
self._socket.bind(url)
self._logger = logging.getLogger('NanomsgPublisher')
def publish(self, message):
""" Publishes message
Args:
message (String): Message to publish
"""
self._socket.send('{}|{}'.format(self._topic, message))
def __enter__(self):
""" Statement used for the `` with ... as ...:`` returns
the object to use in the ``with`` block
Returns:
NanomsgPublisher
"""
return self
def __exit__(self, exc_type, exc_value, traceback):
""" Executed when leaving ``with`` block, regardless whether
because of an exception or normal program flow
"""
self._socket.close()
