class TestGeneralSocketMethods(unittest.TestCase): def setUp(self): self.socket = Socket(PAIR) def tearDown(self): self.socket.close() def test_bind(self): endpoint = self.socket.bind(SOCKET_ADDRESS) self.assertNotEqual(None, endpoint) def test_connect(self): endpoint = self.socket.connect(SOCKET_ADDRESS) self.assertNotEqual(None, endpoint) def test_is_open_is_true_when_open(self): self.assertTrue(self.socket.is_open()) def test_is_open_is_false_when_closed(self): self.socket.close() self.assertFalse(self.socket.is_open()) def test_set_and_get_int_option(self): expected = 500 self.socket.set_int_option(SOL_SOCKET, SNDBUF, expected) actual = self.socket.get_int_option(SOL_SOCKET, SNDBUF) self.assertEqual(expected, actual)
def nano_server(endpoint): s = Socket(PAIR) s.bind('ipc://' + endpoint) #s.bind('tcp://127.0.0.1:9001') try: while True: s.recv() s.send('ok') finally: s.close()
class NanomsgSubscriber(HiddenSubscriber): """ Subscriber class subscribing to a certain topic Attributes: context (zmq.Context): socket (Socket): Socket object of ZMQ context topic (String): Topic subscriber subscribes to """ def __init__(self, url, topic): """ Initializes object Args: url (String): url to publish messages to topic (String): Topic to publish messages under """ super(NanomsgSubscriber, self).__init__(topic) self._socket = Socket(SUB) self._socket.recv_timeout = 500 # Wait 500ms for receiving msgs self._socket.connect(url) self._socket.set_string_option(SUB, SUB_SUBSCRIBE, topic + '|') self._logger = logging.getLogger('NanomsgSubscriber') def receive(self): """ Receives a message Returns: String """ message = self._socket.recv() return message[len(self.topic) + 1:] def __enter__(self): """ Statement used for the `` with ... as ...:`` returns the object to use in the ``with`` block Returns: NanomsgSubscriber """ return self def __exit__(self, exc_type, exc_value, traceback): """ Executed when leaving ``with`` block, regardless whether because of an exception or normal program flow """ self._socket.close()
class ConsoleProxy(object): def __init__(self, url, recv_timeout=1000, send_timeout=1000): self.url = url.encode() self.socket = Socket(REQ) self.socket.recv_timeout = recv_timeout self.socket.send_timeout = send_timeout super(ConsoleProxy, self).__init__() def connect(self): self.socket.connect(self.url) def disconnect(self): self.socket.close() def send_command(self, cmd_name, echo=True, **kwargs): command = { "name": cmd_name, "args": kwargs } dump = msgpack.packb(command) if echo: print('send: ', command) try: self.socket.send(dump) except NanoMsgAPIError as e: print('send_error', e.msg) pass try: recv = self.socket.recv() unpack_msg = msgpack.unpackb(recv, encoding='utf-8') if echo: print('recv: ', recv) print(unpack_msg) return unpack_msg except NanoMsgAPIError as e: print('recv_error', e.msg)
def main(): sift_root = init.env_var_or_exit('SIFT_ROOT') ipc_root = init.env_var_or_exit('IPC_ROOT') dag = load_dag(sift_root) threads = {} sockets = [] node_indexes = sys.argv[1:] if len(node_indexes) == 0: print('no nodes to execute') return 1 dry = os.environ.get('DRY', 'false') if dry == 'true': return 0 for i in map(int, node_indexes): src = os.path.join(sift_root, dag['dag']['nodes'][i]['implementation']['python']) print('loading ' + src) m = new_module(i, src) # Create nanomsg socket. addr = 'ipc://%s/%d.sock'% (ipc_root, i) s = Socket(REP) s.recv_max_size = -1 s.connect(addr) print('connected to '+ addr) sockets.append(s) # Launch request handler. t = threading.Thread(target=listen_and_reply, args=(s, m.compute)) t.daemon = True t.start() threads[i] = t try: while True: time.sleep(1) for i, thr in threads.items(): if not thr.isAlive(): raise Exception('thread of node with index %d is dead' % i) finally: print('closing sockets') for s in sockets: s.close()
class Client(Common): def __init__(self, uri="inproc://server", id=None): self.sock = Socket(REQ) self.sock.connect(uri) self.id = id def close(self): self.sock.close() def call(self, func, *args): msg = self.dump((func, args)) self.sock.send(msg) return self.load(self.sock.recv()) def sync_call(self, func, *args): id = self.call(func, *args) print "got id", id while True: res = self.getresult(id) if res: return res time.sleep(0.1) def result(self, id, result): return self.call("result", id, result) def getresult(self, id): return self.call("getresult", id) def getstate(self, key): return self.call("getstate", key) def setstate(self, key, value): return self.call("setstate", key, value) def getlog(self, id, since=0): return self.call("getlog", id, since) def out(self, msg): return self.call("out", self.id, msg) def err(self, msg): return self.call("err", self.id, msg)
class NanomsgPublisher(HiddenPublisher): """ Publisher class publishing messages to a certain topic to an url Attributes: context (zmq.Context): socket (Socket): Socket object of ZMQ context topic (String): Topic publisher publishs to """ def __init__(self, url, topic): """ Initializes object Args: url (String): url to publish messages to topic (String): Topic to publish messages under """ super(NanomsgPublisher, self).__init__(topic) self._socket = Socket(PUB) self._socket.send_timeout = 1 # Wait 1ms for sending self._socket.bind(url) self._logger = logging.getLogger('NanomsgPublisher') def publish(self, message): """ Publishes message Args: message (String): Message to publish """ self._socket.send('{}|{}'.format(self._topic, message)) def __enter__(self): """ Statement used for the `` with ... as ...:`` returns the object to use in the ``with`` block Returns: NanomsgPublisher """ return self def __exit__(self, exc_type, exc_value, traceback): """ Executed when leaving ``with`` block, regardless whether because of an exception or normal program flow """ self._socket.close()
class ServiceDiscovery(object): def __init__(self, port, deadline=5000): self.socket = Socket(SURVEYOR) self.port = port self.deadline = deadline self.services = defaultdict(set) def bind(self): self.socket.bind('tcp://172.30.42.174:%s' % self.port) self.socket.set_int_option(SURVEYOR, SURVEYOR_DEADLINE, self.deadline) def discover(self): if not self.socket.is_open(): return self.services self.services = defaultdict(set) self.socket.send('service query') while True: try: response = self.socket.recv() except NanoMsgAPIError: break service, address = response.split('|') self.services[service].add(address) return self.services def resolve(self, service): providers = self.services[service] if not providers: return None return random.choice(tuple(providers)) def close(self): self.socket.close()
def nano_client(endpoint): s = Socket(PAIR) s.connect('ipc://' + endpoint) #s.connect('tcp://127.0.0.1:9001') count = 0 payload = test_str(SIZE) start = time.time() try: while True: s.send(payload) s.recv() count += 1 except: pass finally: end = time.time() total = end - start print 'total: ', count print 'took: ', total print 'req / sec:', count / total print 'bandwidth: %f MBps' % (((count / total) * SIZE) / 2 ** 20) s.close()
for index in range(0, 10): print("Consumer gevent.socket.wait_read") wait_read(s1.recv_fd) print("Consumer gevent.socket.wait_read finished") req = s1.recv(flags=DONTWAIT) s1.send("{} pong {}".format(req, index)) def provider(): for index in range(0, 10): print("Provider send ping...") req = s2.send("ping") print("Provider gevent.socket.wait_read") wait_read(s2.recv_fd) print("Provider gevent.socket.wait_read finished") rep = s2.recv(flags=DONTWAIT) print("Provider receive: {}".format(rep)) try: c = gevent.spawn(consumer) p = gevent.spawn(provider) print("Waiting consumer and provider...") gevent.joinall([c, p]) finally: s1.close() s2.close()
# vim: set fileencoding=utf-8 from nanomsg import Socket, PULL URL = "ipc:///tmp/example2" socket = Socket(PULL) print("Socket created") socket.bind(URL) print("Bound to URL {}".format(URL)) while True: message = socket.recv() print("Received: {}".format(message)) socket.close() print("Socket closed")
class PullWorker(threading.Thread): """ End point of pipeline """ def __init__(self, url, event): """ Initializes object Args: url (String): Url to bind to """ super(PullWorker, self).__init__() self._socket = Socket(PULL) self._url = url self._socket.bind(self._url) self._queue = Queue.Queue() self._stop = event def receive(self): """ Reveives message """ raw_msg = self._socket.recv() parsed = {} try: parsed = json.loads(raw_msg) except: logging.exception( 'PullWorker: Received msg could not be JSON parsed %s' % raw_msg) return parsed def run(self): """ Thread run method """ while not self._stop.is_set(): dic = self.receive() self._queue.put(dic) self.__exit__() def next_message(self): """ Return next message of queue Returns: message: Dictionary representing send message """ msg = self._queue.get() self._queue.task_done() return msg def __enter__(self): """ Statement used for the `` with ... as ...:`` returns the object to use in the ``with`` block Returns: PullWorker """ return self def __exit__(self, exc_type, exc_value, traceback): """ Executed when leaving ``with`` block, regardless whether because of an exception or normal program flow """ self._socket.close()
class PushWorker(threading.Thread): """ Pushes data into pipeline Attributes: url: String specifying endpoint socket connects to. topic: String specifying topic worker is relying data to samplingrate: Integer specifying the rate at which samples are collected in Herz [1/s] socket: Nanomsg socket object stop: threading.Event to stop Thread Note: url must be of the form ``<protocol>://<address>`` where ``<protocol>`` can have to following values: - **tcp** - send data over computer network using tcp protocol. In this case ``<address>`` has to be a valid IPv4 address of the form ``<ipv4>:<port>`` - **inproc** - Share data between threads of the same process using shared memory. In this case ``<address>`` can be an arbitrary string. - **ipc** - Share data between two processes using a file. In this case ``<address>`` can be an arbitrary string. """ def __init__(self, url, topic, loggername, samplingrate=100, retry_interval=1): """ Initializes object Args: url (string): specifying endpoint socket connects to. topic (String): specifying topic worker is relying data to. samplingrate (int): specifying the rate at which samples are collected in Herz [1/s]. """ super(PushWorker, self).__init__() self._samplingrate = samplingrate self._socket = Socket(PUSH) self._url = url self._topic = topic self._socket.connect(self._url) self._logger = logging.getLogger(loggername) self._name = loggername self._stop = threading.Event() def send(self, data): """ Sends message into queue """ message = json.dumps({'topic': self._topic, 'data': data}) self._socket.send(message) def _get_data_point(self): """ Message to retrieve one datapoint """ raise NotImplementedError('Method get_data_point not implemented for ' + \ 'PushWorker') def run(self): """ Run method for thread """ for datapoint in self._get_data_point(): self.send(datapoint) self._logger.debug('topic: {} value: {} target: {}'.format( self._topic, datapoint, self._url)) time.sleep(1. / self._samplingrate) if self.stopped: self._logger.info('{} is going down...'.format(self._name)) break self.__exit__() def stop(self): """ Stops thread """ self._stop.set() @property def stopped(self): """ Checks whether Thread is stopped. Returns True if thread stopped else False """ return self._stop.is_set() def __enter__(self): """ Statement used for the `` with ... as ...:`` returns the object to use in the ``with`` block Returns: PushWorker """ return self def __exit__(self, exc_type, exc_value, traceback): """ Executed when leaving ``with`` block, regardless whether because of an exception or normal program flow """ self._socket.close()
from __future__ import print_function from nanomsg import Socket, PAIR, PUB s1 = Socket(PAIR) s2 = Socket(PAIR) s1.bind('inproc://bob') s2.connect('inproc://bob') s1.send(b'hello nanomsg') print(s2.recv()) s1.close() s2.close() with Socket(PAIR) as s3: with Socket(PAIR) as s4: s3.bind('inproc://demo') s4.connect('inproc://demo') s3.send('hi, I use "with"') print(s4.recv()) s4.send('Ok, I see.') print(s3.recv())
def run(self, stopflag, q): # signal.signal(signal.SIGINT, signal.SIG_IGN) qO, qN, qD = q print('tcp://*:' + self.port) s1 = Socket(REP) # s1.recv_buffer_size=1024*1024 # s1.reconnect_interval_max=1000 s1.bind('tcp://*:' + self.port) pb_n = args_pb2.p_n() s_n = self.s_n ps_n = s_n * (s_n + 1) pb_n.s_n = s_n # print("nanoMesg up") time.sleep(0.5) running = True pb_sid = args_pb2.p_sid() pb_sid.sid = 0 clients = dict() sys_platform = sys.platform while (running): # print("loop start") if stopflag.value > len(clients.keys()): running = False break if sys_platform == 'win32': try: recvstr = s1.recv() except KeyboardInterrupt: if len(clients.keys()) <= 0: running = False elif stopflag.value >= 0: print("stopflag.value: ", stopflag.value) with stopflag.get_lock(): stopflag.value += 1 continue # os.kill(self.ppid, signal.CTRL_C_EVENT) else: recvstr = s1.recv() # recvstr=s1.recv() # print(ord('c'),ord('p'),ord('r'),ord('k'),"recvstr[0]: ",recvstr[0]) if recvstr[0] == ord('c'): pb_cap = args_pb2.p_cap() pb_cap.ParseFromString(recvstr[1:]) # print("pb_cap.idx",pb_cap.idx) s1.send(pb_n.SerializeToString()) elif recvstr[0] == ord('p'): pb_gpuid = args_pb2.p_str() pb_gpuid.ParseFromString(recvstr[1:]) # TODO 解析 p_str get ohist if pb_gpuid.histNum > 0: # print(list(pb_gpuid.ohist)) qD.put( (pb_gpuid.histNum, self.bestcs, list(pb_gpuid.ohist))) pb_ga = args_pb2.p_ga() pb_ga.start = 0 pb_ga.stop = -1 ii = -1 ind_ = [] try: ii, ind_, self.bestcs = qO.get( False ) #TODO change proto to allow ord('p') fail and retry later. except queue.Empty: if stopflag.value >= 1: print("stopflag.value: ", stopflag.value) pb_ga.idx = -2 stopflag.value += 1 else: pb_ga.idx = -1 s1.send(pb_ga.SerializeToString()) continue # ind_=[random.random()]*(s_n*(s_n+1)) pb_ga.idx = ii pb_ga.bestfv = self.bestcs for i in range(s_n): pb_ga.params.append(ind_[i]) for i in range(s_n * s_n - s_n): pb_ga.params.append(ind_[i + s_n]) for i in range(s_n): pb_ga.params.append(ind_[i + s_n * s_n]) s1.send(pb_ga.SerializeToString()) # clients_lastTimeS[pb_ga.idx]=time.perf_counter() # print("p: ",clients_lastTimeS[pb_ga.idx]) if pb_gpuid.str not in clients: clients[pb_gpuid.str] = gpuClient() print(pb_gpuid.str, " connected.") else: clients[pb_gpuid.str].updateTimeStamp() elif recvstr[0] == ord('r'): res = args_pb2.res() res.ParseFromString(recvstr[1:]) # print("res chi2 recv",res.e, " ridx: ",res.ridx) if stopflag.value >= 1: pb_sid.sid = -1 print("time spend: ", clients.pop(res.idx).runTime) if len(clients.keys()) <= 0: running = False # print("stopflag: ",stopflag.value) # s1.close() s1.send(pb_sid.SerializeToString()) # print("pb_sid send",pb_sid.sid, " ridx: ",res.ridx) if res.idx in clients: clients[res.idx].updateRunTime() #TODO 解析res get shist qN.put((res.ridx, res.e)) if res.hist: # self.counter=self.counter+1 # print(self.counter,self.bestcs) if self.bestcs < 3.2E31: qD.put((-1, self.bestcs, list(res.shist))) elif recvstr[0] == ord('k'): pidx = args_pb2.p_sid() pidx.ParseFromString(recvstr[1:]) # print("keepalive: ",pidx.sid) s1.send('l') else: print("Err recv: ", recvstr[0]) s1.close() qN.close() qD.close() qN.join_thread() qD.join_thread() print("paramsServ done!")
from __future__ import print_function from nanomsg import Socket, PAIR, PUB import time s1 = Socket(PAIR) s1.bind('ipc:///tmp/wand1.ipc') while (True): kkk = "hello world!" print(kkk) s1.send(kkk) time.sleep(1) s1.close()
class PubHandler(logging.Handler): """ A basic logging handler that emits log messages through a PUB socket. Takes a PUB socket already bound to interfaces or an interface to bind to. Example:: sock = context.socket(PUB) sock.bind("inproc://log") handler = PUBHandler(sock) Or:: handler = PUBHandler("inproc://log") These are equivalent. Log messages handled by this handler are broadcase with nanomsg PUB/SUB model. """ dayfmt = '%Y %m %d %H:%M:%S' formatters = { logging.DEBUG: EncodingFormatter( "%(asctime)-15s %(name)s.%(levelname)s " + # "in %(module)s [%(pathname)s:%(lineno)d]" + "- %(message)s\n", dayfmt), logging.INFO: EncodingFormatter( "%(asctime)-15s %(name)s.%(levelname)s " + # "in %(module)s [%(pathname)s:%(lineno)d]" + "- %(message)s\n", dayfmt), logging.WARN: EncodingFormatter( "%(asctime)-15s %(name)s.%(levelname)s " + # "in %(module)s [%(pathname)s:%(lineno)d]" + "- %(message)s\n", dayfmt), logging.ERROR: EncodingFormatter( "%(asctime)-15s %(name)s.%(levelname)s " + # "in %(module)s [%(pathname)s:%(lineno)d]" + "- %(message)s - %(exc_info)s\n", dayfmt), logging.CRITICAL: EncodingFormatter( "%(asctime)-15s %(name)s.%(levelname)s " + # "in %(module)s [%(pathname)s:%(lineno)d]" + "- %(message)s\n", dayfmt), } def __init__(self, pub_addr): logging.Handler.__init__(self) self.sock = Socket(PUB) self.sock.connect(pub_addr) log_level = logging.DEBUG self.setFormatter(PubHandler.formatters[log_level]) def emit(self, record): """ Emit a log message on nanomsg pub/sub model. """ #TODO is send asynchronous ? try: self.sock.send(self.format(record)) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def colse(self): self.sock.close() logging.Handler.close(self) def __del__(self): self.sock.close()