class TestGeneralSocketMethods(unittest.TestCase):
def setUp(self):
self.socket = Socket(PAIR)
def tearDown(self):
self.socket.close()
def test_bind(self):
endpoint = self.socket.bind(SOCKET_ADDRESS)
self.assertNotEqual(None, endpoint)
def test_connect(self):
endpoint = self.socket.connect(SOCKET_ADDRESS)
self.assertNotEqual(None, endpoint)
def test_is_open_is_true_when_open(self):
self.assertTrue(self.socket.is_open())
def test_is_open_is_false_when_closed(self):
self.socket.close()
self.assertFalse(self.socket.is_open())
def test_set_and_get_int_option(self):
expected = 500
self.socket.set_int_option(SOL_SOCKET, SNDBUF, expected)
actual = self.socket.get_int_option(SOL_SOCKET, SNDBUF)
self.assertEqual(expected, actual)
def nano_server(endpoint):
s = Socket(PAIR)
s.bind('ipc://' + endpoint)
#s.bind('tcp://127.0.0.1:9001')
try:
while True:
s.recv()
s.send('ok')
finally:
s.close()
class NanomsgSubscriber(HiddenSubscriber):
""" Subscriber class subscribing to a certain topic
Attributes:
context (zmq.Context):
socket (Socket): Socket object of ZMQ context
topic (String): Topic subscriber subscribes to
"""
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgSubscriber, self).__init__(topic)
self._socket = Socket(SUB)
self._socket.recv_timeout = 500 # Wait 500ms for receiving msgs
self._socket.connect(url)
self._socket.set_string_option(SUB, SUB_SUBSCRIBE, topic + '|')
self._logger = logging.getLogger('NanomsgSubscriber')
def receive(self):
""" Receives a message
Returns:
String
"""
message = self._socket.recv()
return message[len(self.topic) + 1:]
def __enter__(self):
""" Statement used for the `` with ... as ...:`` returns
the object to use in the ``with`` block
Returns:
NanomsgSubscriber
"""
return self
def __exit__(self, exc_type, exc_value, traceback):
""" Executed when leaving ``with`` block, regardless whether
because of an exception or normal program flow
"""
self._socket.close()
class ConsoleProxy(object):
def __init__(self, url, recv_timeout=1000, send_timeout=1000):
self.url = url.encode()
self.socket = Socket(REQ)
self.socket.recv_timeout = recv_timeout
self.socket.send_timeout = send_timeout
super(ConsoleProxy, self).__init__()
def connect(self):
self.socket.connect(self.url)
def disconnect(self):
self.socket.close()
def send_command(self, cmd_name, echo=True, **kwargs):
command = {
"name": cmd_name,
"args": kwargs
}
dump = msgpack.packb(command)
if echo:
print('send: ', command)
try:
self.socket.send(dump)
except NanoMsgAPIError as e:
print('send_error', e.msg)
pass
try:
recv = self.socket.recv()
unpack_msg = msgpack.unpackb(recv, encoding='utf-8')
if echo:
print('recv: ', recv)
print(unpack_msg)
return unpack_msg
except NanoMsgAPIError as e:
print('recv_error', e.msg)
def main():
sift_root = init.env_var_or_exit('SIFT_ROOT')
ipc_root = init.env_var_or_exit('IPC_ROOT')
dag = load_dag(sift_root)
threads = {}
sockets = []
node_indexes = sys.argv[1:]
if len(node_indexes) == 0:
print('no nodes to execute')
return 1
dry = os.environ.get('DRY', 'false')
if dry == 'true':
return 0
for i in map(int, node_indexes):
src = os.path.join(sift_root, dag['dag']['nodes'][i]['implementation']['python'])
print('loading ' + src)
m = new_module(i, src)
# Create nanomsg socket.
addr = 'ipc://%s/%d.sock'% (ipc_root, i)
s = Socket(REP)
s.recv_max_size = -1
s.connect(addr)
print('connected to '+ addr)
sockets.append(s)
# Launch request handler.
t = threading.Thread(target=listen_and_reply, args=(s, m.compute))
t.daemon = True
t.start()
threads[i] = t
try:
while True:
time.sleep(1)
for i, thr in threads.items():
if not thr.isAlive():
raise Exception('thread of node with index %d is dead' % i)
finally:
print('closing sockets')
for s in sockets: s.close()
class Client(Common):
def __init__(self, uri="inproc://server", id=None):
self.sock = Socket(REQ)
self.sock.connect(uri)
self.id = id
def close(self):
self.sock.close()
def call(self, func, *args):
msg = self.dump((func, args))
self.sock.send(msg)
return self.load(self.sock.recv())
def sync_call(self, func, *args):
id = self.call(func, *args)
print "got id", id
while True:
res = self.getresult(id)
if res:
return res
time.sleep(0.1)
def result(self, id, result):
return self.call("result", id, result)
def getresult(self, id):
return self.call("getresult", id)
def getstate(self, key):
return self.call("getstate", key)
def setstate(self, key, value):
return self.call("setstate", key, value)
def getlog(self, id, since=0):
return self.call("getlog", id, since)
def out(self, msg):
return self.call("out", self.id, msg)
def err(self, msg):
return self.call("err", self.id, msg)
class NanomsgPublisher(HiddenPublisher):
""" Publisher class publishing messages to a certain topic to an url
Attributes:
context (zmq.Context):
socket (Socket): Socket object of ZMQ context
topic (String): Topic publisher publishs to
"""
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(NanomsgPublisher, self).__init__(topic)
self._socket = Socket(PUB)
self._socket.send_timeout = 1 # Wait 1ms for sending
self._socket.bind(url)
self._logger = logging.getLogger('NanomsgPublisher')
def publish(self, message):
""" Publishes message
Args:
message (String): Message to publish
"""
self._socket.send('{}|{}'.format(self._topic, message))
def __enter__(self):
""" Statement used for the `` with ... as ...:`` returns
the object to use in the ``with`` block
Returns:
NanomsgPublisher
"""
return self
def __exit__(self, exc_type, exc_value, traceback):
""" Executed when leaving ``with`` block, regardless whether
because of an exception or normal program flow
"""
self._socket.close()
class ServiceDiscovery(object):
def __init__(self, port, deadline=5000):
self.socket = Socket(SURVEYOR)
self.port = port
self.deadline = deadline
self.services = defaultdict(set)
def bind(self):
self.socket.bind('tcp://172.30.42.174:%s' % self.port)
self.socket.set_int_option(SURVEYOR, SURVEYOR_DEADLINE, self.deadline)
def discover(self):
if not self.socket.is_open():
return self.services
self.services = defaultdict(set)
self.socket.send('service query')
while True:
try:
response = self.socket.recv()
except NanoMsgAPIError:
break
service, address = response.split('|')
self.services[service].add(address)
return self.services
def resolve(self, service):
providers = self.services[service]
if not providers:
return None
return random.choice(tuple(providers))
def close(self):
self.socket.close()
def nano_client(endpoint):
s = Socket(PAIR)
s.connect('ipc://' + endpoint)
#s.connect('tcp://127.0.0.1:9001')
count = 0
payload = test_str(SIZE)
start = time.time()
try:
while True:
s.send(payload)
s.recv()
count += 1
except:
pass
finally:
end = time.time()
total = end - start
print 'total: ', count
print 'took: ', total
print 'req / sec:', count / total
print 'bandwidth: %f MBps' % (((count / total) * SIZE) / 2 ** 20)
s.close()
for index in range(0, 10):
print("Consumer gevent.socket.wait_read")
wait_read(s1.recv_fd)
print("Consumer gevent.socket.wait_read finished")
req = s1.recv(flags=DONTWAIT)
s1.send("{} pong {}".format(req, index))
def provider():
for index in range(0, 10):
print("Provider send ping...")
req = s2.send("ping")
print("Provider gevent.socket.wait_read")
wait_read(s2.recv_fd)
print("Provider gevent.socket.wait_read finished")
rep = s2.recv(flags=DONTWAIT)
print("Provider receive: {}".format(rep))
try:
c = gevent.spawn(consumer)
p = gevent.spawn(provider)
print("Waiting consumer and provider...")
gevent.joinall([c, p])
finally:
s1.close()
s2.close()
# vim: set fileencoding=utf-8
from nanomsg import Socket, PULL
URL = "ipc:///tmp/example2"
socket = Socket(PULL)
print("Socket created")
socket.bind(URL)
print("Bound to URL {}".format(URL))
while True:
message = socket.recv()
print("Received: {}".format(message))
socket.close()
print("Socket closed")
class PullWorker(threading.Thread):
""" End point of pipeline
"""
def __init__(self, url, event):
""" Initializes object
Args:
url (String): Url to bind to
"""
super(PullWorker, self).__init__()
self._socket = Socket(PULL)
self._url = url
self._socket.bind(self._url)
self._queue = Queue.Queue()
self._stop = event
def receive(self):
""" Reveives message
"""
raw_msg = self._socket.recv()
parsed = {}
try:
parsed = json.loads(raw_msg)
except:
logging.exception(
'PullWorker: Received msg could not be JSON parsed %s' %
raw_msg)
return parsed
def run(self):
""" Thread run method
"""
while not self._stop.is_set():
dic = self.receive()
self._queue.put(dic)
self.__exit__()
def next_message(self):
""" Return next message of queue
Returns:
message: Dictionary representing send message
"""
msg = self._queue.get()
self._queue.task_done()
return msg
def __enter__(self):
""" Statement used for the `` with ... as ...:`` returns
the object to use in the ``with`` block
Returns:
PullWorker
"""
return self
def __exit__(self, exc_type, exc_value, traceback):
""" Executed when leaving ``with`` block, regardless whether
because of an exception or normal program flow
"""
self._socket.close()
class PushWorker(threading.Thread):
""" Pushes data into pipeline
Attributes:
url: String specifying endpoint socket connects to.
topic: String specifying topic worker is relying data to
samplingrate: Integer specifying the rate at which samples are
collected in Herz [1/s]
socket: Nanomsg socket object
stop: threading.Event to stop Thread
Note:
url must be of the form ``<protocol>://<address>`` where
``<protocol>`` can have to following values:
- **tcp** - send data over computer network using tcp protocol. In
this case ``<address>`` has to be a valid IPv4 address of the
form ``<ipv4>:<port>``
- **inproc** - Share data between threads of the same process using
shared memory. In this case ``<address>`` can be an arbitrary string.
- **ipc** - Share data between two processes using a file. In
this case ``<address>`` can be an arbitrary string.
"""
def __init__(self,
url,
topic,
loggername,
samplingrate=100,
retry_interval=1):
""" Initializes object
Args:
url (string): specifying endpoint socket connects to.
topic (String): specifying topic worker is relying data to.
samplingrate (int): specifying the rate at which samples are
collected in Herz [1/s].
"""
super(PushWorker, self).__init__()
self._samplingrate = samplingrate
self._socket = Socket(PUSH)
self._url = url
self._topic = topic
self._socket.connect(self._url)
self._logger = logging.getLogger(loggername)
self._name = loggername
self._stop = threading.Event()
def send(self, data):
""" Sends message into queue
"""
message = json.dumps({'topic': self._topic, 'data': data})
self._socket.send(message)
def _get_data_point(self):
""" Message to retrieve one datapoint
"""
raise NotImplementedError('Method get_data_point not implemented for ' + \
'PushWorker')
def run(self):
""" Run method for thread
"""
for datapoint in self._get_data_point():
self.send(datapoint)
self._logger.debug('topic: {} value: {} target: {}'.format(
self._topic, datapoint, self._url))
time.sleep(1. / self._samplingrate)
if self.stopped:
self._logger.info('{} is going down...'.format(self._name))
break
self.__exit__()
def stop(self):
""" Stops thread
"""
self._stop.set()
@property
def stopped(self):
""" Checks whether Thread is stopped.
Returns
True if thread stopped else False
"""
return self._stop.is_set()
def __enter__(self):
""" Statement used for the `` with ... as ...:`` returns
the object to use in the ``with`` block
Returns:
PushWorker
"""
return self
def __exit__(self, exc_type, exc_value, traceback):
""" Executed when leaving ``with`` block, regardless whether
because of an exception or normal program flow
"""
self._socket.close()
from __future__ import print_function
from nanomsg import Socket, PAIR, PUB
s1 = Socket(PAIR)
s2 = Socket(PAIR)
s1.bind('inproc://bob')
s2.connect('inproc://bob')
s1.send(b'hello nanomsg')
print(s2.recv())
s1.close()
s2.close()
with Socket(PAIR) as s3:
with Socket(PAIR) as s4:
s3.bind('inproc://demo')
s4.connect('inproc://demo')
s3.send('hi, I use "with"')
print(s4.recv())
s4.send('Ok, I see.')
print(s3.recv())
def run(self, stopflag, q):
# signal.signal(signal.SIGINT, signal.SIG_IGN)
qO, qN, qD = q
print('tcp://*:' + self.port)
s1 = Socket(REP)
# s1.recv_buffer_size=1024*1024
# s1.reconnect_interval_max=1000
s1.bind('tcp://*:' + self.port)
pb_n = args_pb2.p_n()
s_n = self.s_n
ps_n = s_n * (s_n + 1)
pb_n.s_n = s_n
# print("nanoMesg up")
time.sleep(0.5)
running = True
pb_sid = args_pb2.p_sid()
pb_sid.sid = 0
clients = dict()
sys_platform = sys.platform
while (running):
# print("loop start")
if stopflag.value > len(clients.keys()):
running = False
break
if sys_platform == 'win32':
try:
recvstr = s1.recv()
except KeyboardInterrupt:
if len(clients.keys()) <= 0:
running = False
elif stopflag.value >= 0:
print("stopflag.value: ", stopflag.value)
with stopflag.get_lock():
stopflag.value += 1
continue
# os.kill(self.ppid, signal.CTRL_C_EVENT)
else:
recvstr = s1.recv()
# recvstr=s1.recv()
# print(ord('c'),ord('p'),ord('r'),ord('k'),"recvstr[0]: ",recvstr[0])
if recvstr[0] == ord('c'):
pb_cap = args_pb2.p_cap()
pb_cap.ParseFromString(recvstr[1:])
# print("pb_cap.idx",pb_cap.idx)
s1.send(pb_n.SerializeToString())
elif recvstr[0] == ord('p'):
pb_gpuid = args_pb2.p_str()
pb_gpuid.ParseFromString(recvstr[1:])
# TODO 解析 p_str get ohist
if pb_gpuid.histNum > 0:
# print(list(pb_gpuid.ohist))
qD.put(
(pb_gpuid.histNum, self.bestcs, list(pb_gpuid.ohist)))
pb_ga = args_pb2.p_ga()
pb_ga.start = 0
pb_ga.stop = -1
ii = -1
ind_ = []
try:
ii, ind_, self.bestcs = qO.get(
False
) #TODO change proto to allow ord('p') fail and retry later.
except queue.Empty:
if stopflag.value >= 1:
print("stopflag.value: ", stopflag.value)
pb_ga.idx = -2
stopflag.value += 1
else:
pb_ga.idx = -1
s1.send(pb_ga.SerializeToString())
continue
# ind_=[random.random()]*(s_n*(s_n+1))
pb_ga.idx = ii
pb_ga.bestfv = self.bestcs
for i in range(s_n):
pb_ga.params.append(ind_[i])
for i in range(s_n * s_n - s_n):
pb_ga.params.append(ind_[i + s_n])
for i in range(s_n):
pb_ga.params.append(ind_[i + s_n * s_n])
s1.send(pb_ga.SerializeToString())
# clients_lastTimeS[pb_ga.idx]=time.perf_counter()
# print("p: ",clients_lastTimeS[pb_ga.idx])
if pb_gpuid.str not in clients:
clients[pb_gpuid.str] = gpuClient()
print(pb_gpuid.str, " connected.")
else:
clients[pb_gpuid.str].updateTimeStamp()
elif recvstr[0] == ord('r'):
res = args_pb2.res()
res.ParseFromString(recvstr[1:])
# print("res chi2 recv",res.e, " ridx: ",res.ridx)
if stopflag.value >= 1:
pb_sid.sid = -1
print("time spend: ", clients.pop(res.idx).runTime)
if len(clients.keys()) <= 0:
running = False
# print("stopflag: ",stopflag.value)
# s1.close()
s1.send(pb_sid.SerializeToString())
# print("pb_sid send",pb_sid.sid, " ridx: ",res.ridx)
if res.idx in clients:
clients[res.idx].updateRunTime()
#TODO 解析res get shist
qN.put((res.ridx, res.e))
if res.hist:
# self.counter=self.counter+1
# print(self.counter,self.bestcs)
if self.bestcs < 3.2E31:
qD.put((-1, self.bestcs, list(res.shist)))
elif recvstr[0] == ord('k'):
pidx = args_pb2.p_sid()
pidx.ParseFromString(recvstr[1:])
# print("keepalive: ",pidx.sid)
s1.send('l')
else:
print("Err recv: ", recvstr[0])
s1.close()
qN.close()
qD.close()
qN.join_thread()
qD.join_thread()
print("paramsServ done!")
from __future__ import print_function
from nanomsg import Socket, PAIR, PUB
import time
s1 = Socket(PAIR)
s1.bind('ipc:///tmp/wand1.ipc')
while (True):
kkk = "hello world!"
print(kkk)
s1.send(kkk)
time.sleep(1)
s1.close()
class PubHandler(logging.Handler):
"""
A basic logging handler that emits log messages through a PUB socket.
Takes a PUB socket already bound to interfaces or an interface to bind to.
Example::
sock = context.socket(PUB)
sock.bind("inproc://log")
handler = PUBHandler(sock)
Or::
handler = PUBHandler("inproc://log")
These are equivalent.
Log messages handled by this handler are broadcase with nanomsg PUB/SUB model.
"""
dayfmt = '%Y %m %d %H:%M:%S'
formatters = {
logging.DEBUG:
EncodingFormatter(
"%(asctime)-15s %(name)s.%(levelname)s " +
# "in %(module)s [%(pathname)s:%(lineno)d]" +
"- %(message)s\n",
dayfmt),
logging.INFO:
EncodingFormatter(
"%(asctime)-15s %(name)s.%(levelname)s " +
# "in %(module)s [%(pathname)s:%(lineno)d]" +
"- %(message)s\n",
dayfmt),
logging.WARN:
EncodingFormatter(
"%(asctime)-15s %(name)s.%(levelname)s " +
# "in %(module)s [%(pathname)s:%(lineno)d]" +
"- %(message)s\n",
dayfmt),
logging.ERROR:
EncodingFormatter(
"%(asctime)-15s %(name)s.%(levelname)s " +
# "in %(module)s [%(pathname)s:%(lineno)d]" +
"- %(message)s - %(exc_info)s\n",
dayfmt),
logging.CRITICAL:
EncodingFormatter(
"%(asctime)-15s %(name)s.%(levelname)s " +
# "in %(module)s [%(pathname)s:%(lineno)d]" +
"- %(message)s\n",
dayfmt),
}
def __init__(self, pub_addr):
logging.Handler.__init__(self)
self.sock = Socket(PUB)
self.sock.connect(pub_addr)
log_level = logging.DEBUG
self.setFormatter(PubHandler.formatters[log_level])
def emit(self, record):
""" Emit a log message on nanomsg pub/sub model. """
#TODO is send asynchronous ?
try:
self.sock.send(self.format(record))
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def colse(self):
self.sock.close()
logging.Handler.close(self)
def __del__(self):
self.sock.close()
