def send(self, message_type=None, message_type_args={}, request_id=None):
if(not request_id):
request_id = uuid.uuid4()
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect("tcp://{0}:{1}".format(self._ip, self._port))
socket.poll(timeout=1)
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
lib.debug.info("Sending request {0} …".format(request_id))
timestarted = time.time()
hostdetails = simplejson.dumps({'hostname':lib.hostname_ip.hostname, 'ip':lib.hostname_ip.ip})
send_msg = simplejson.dumps(self.process(message_type, message_type_args,hostdetails))
socket.send_multipart([bytes(unicode(request_id)), bytes(unicode(hostdetails)), bytes(unicode(message_type)), bytes(unicode(send_msg))])
while(True):
sockets = dict(poller.poll(10000))
if(sockets):
for s in sockets.keys():
if(sockets[s] == zmq.POLLIN):
try:
(recv_id, recv_hostdetails, recv_msg_type, recved_msg) = s.recv_multipart()
recv_message = self.process(recv_msg_type, recved_msg,recv_hostdetails)
lib.debug.info("Received reply %s : %s [ %s ]" % (recv_id, recv_message, time.time() - timestarted))
except:
lib.debug.info (sys.exc_info())
break
break
lib.debug.info ("Reciever Timeout error : Check if the server is running")
socket.close()
context.term()
def _worker(self,worker_url, worker_id=uuid.uuid4()):
if (sys.platform.lower().find("linux") >= 0):
setproctitle.setproctitle("server-worker")
lib.debug.info (worker_url)
context = zmq.Context()
# Socket to talk to dispatcher
socket = context.socket(zmq.REP)
socket.poll(timeout=1)
socket.connect(worker_url)
while True:
(request_id_rep, state_name_rep, topic_rep, msg_rep) = socket.recv_multipart()
rep_sock.send_multipart([request_id_rep, state_name, msg_rep])
lib.debug.debug(msg_rep)
try:
msg_reved = simplejson.loads(msg_rep)
if (msg_reved['status'] == "free"):
hosts_recieved[msg_reved['hostid']] = msg_reved
lib.debug.debug("sending state : " + state_name_rep + " : ")
else:
return (msg_reved['status'] + " : " + msg_reved['request_id'])
except:
lib.debug.error(str(state_name) + " : " + str(request_id) + " : " + str(sys.exc_info()))
return (str(state_name) + " : " + str(request_id) + " : " + str(sys.exc_info()))
while True:
received = socket.recv_multipart()
lib.debug.info("Received request: [ {0} ] -> [ {1} ]".format(str(worker_id),msg_type_args))
reply = self.process(received)
reply_to_send = simplejson.dumps(reply)
socket.send_multipart([bytes(unicode(hostid)),bytes(unicode(request_id)),bytes(unicode(reply_to_send))])
lib.debug.info("Replied to request: [ {0} ] -> [ {1} ]".format(str(worker_id), msg_type_args))
def send(self, socket, routing=[]):
if hasattr(self, "_remote"):
assert not self._remote, "send() called on a non-local Communicable object."
json_str, buffers = json_encode(self.__dict__)
socket.send_multipart(
routing + [self.__class__.__module__, self.__class__.__name__] + [json_str] + buffers, copy=False
)
def send(self, socket, routing=[]):
if hasattr(self, '_remote'):
assert not self._remote, "send() called on a non-local Communicable object."
json_str, buffers = json_encode(self.__dict__)
socket.send_multipart(
routing + [self.__class__.__module__, self.__class__.__name__] +
[json_str] + buffers,
copy=False)
def run(self):
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind("tcp://*:%s" % port)
time.sleep(0.1)
while True:
msg = socket.recv()
res = self.dispatch_request(msg)
socket.send_multipart(res)
def ban_swabber(ip):
if not zmq:
raise SystemExit("swabber engine enabled but zmq not available!")
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.connect("tcp://127.0.0.1:22620")
socket.send_multipart(("swabber_bans", ip))
socket.close()
return True
def ban_swabber(ip):
if not zmq:
raise SystemExit("swabber engine enabled but zmq not available!")
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.connect("tcp://127.0.0.1:22620")
socket.send_multipart(("swabber_bans", ip))
socket.close()
return True
def send(self, socket, routing=[]):
if hasattr(self, '_remote'):
assert not self._remote, "send() called on a non-local Communicable object."
sendable_dict = dict(
(k, v) for k, v in self.__dict__.items()
if (not k.startswith('_')) and (not callable(self.__dict__[k])))
# replace each buffer with its metadata, and send it separately
buffers = []
encoder = make_CP_encoder(buffers)
json_str = json.dumps(sendable_dict, default=encoder)
socket.send_multipart(
routing + [self.__class__.__module__, self.__class__.__name__] +
[json_str] + buffers,
copy=False)
def send(self, socket, routing=[]):
if hasattr(self, '_remote'):
assert not self._remote, "send() called on a non-local Communicable object."
sendable_dict = dict((k, v) for k, v in self.__dict__.items()
if (not k.startswith('_'))
and (not callable(self.__dict__[k])))
# replace each buffer with its metadata, and send it separately
buffers = []
encoder = make_CP_encoder(buffers)
json_str = json.dumps(sendable_dict, default=encoder)
socket.send_multipart(routing +
[self.__class__.__module__, self.__class__.__name__] +
[json_str] +
buffers, copy=False)
def handler(socket, msg):
ti = str(threading.currentThread().ident)
print ti + "] Handling new message "
#print msg
if len(msg) == 4:
msgfrom = msg[0]
msgtxt = msg[1]
msgid = msg[2]
reply_wanted = msg[3]
print "\tid:" + msgid + " reply_wanted:" + reply_wanted
if reply_wanted == "1":
print "\tthey want a reply. sending it."
socket.send_multipart([msgfrom, 'fooreply', msgid])
def sendmsg(socket, msg, msgid, callback):
"""
Send a message on a socket. Messages are always sent asynchronously.
If a callback is specified, it will be called when a reply is received.
"""
if socket != None and msg != None:
reply_wanted = 0
if callback != None:
reply_wanted = 1
callback_registry_lock.acquire()
callback_registry[msgid] = callback
callback_registry_lock.release()
socket.send_multipart([msg, msgid, str(reply_wanted)])
def watch_batch(self, batch_id):
# Setup a connection to the validator
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.connect(current_app.config['SAWTOOTH_VALIDATOR_URL'])
# Construct the request
request = client_batch_submit_pb2.ClientBatchStatusRequest(
batch_ids=[batch_id], wait=True).SerializeToString()
# Construct the message wrapper
correlation_id = batch_id + uuid.uuid4(
).hex # This must be unique for all in-process requests
msg = Message(correlation_id=correlation_id,
message_type=Message.CLIENT_BATCH_STATUS_REQUEST,
content=request)
# Send the request
socket.send_multipart([msg.SerializeToString()])
# Receive the response
resp = socket.recv_multipart()[-1]
# Parse the message wrapper
msg = Message()
msg.ParseFromString(resp)
# Validate the response type
if msg.message_type != Message.CLIENT_BATCH_STATUS_RESPONSE:
current_app.logger.error("Unexpected response message type")
return
# Parse the response
response = client_batch_submit_pb2.ClientBatchStatusResponse()
response.ParseFromString(msg.content)
# Validate the response status
if response.status != client_batch_submit_pb2.ClientBatchSubmitResponse.OK:
current_app.logger.error("watch batch status failed: {}".format(
response.response_message))
return
# Close the connection to the validator
socket.close()
return client_batch_submit_pb2.ClientBatchStatus.Status.Name(
response.batch_statuses[0].status)
def _reply_heartbeat(self, target):
"""Worker will kill its jobs when it lost connection with the master.
"""
socket = self.ctx.socket(zmq.REP)
socket.linger = 0
socket.setsockopt(zmq.RCVTIMEO,
remote_constants.HEARTBEAT_RCVTIMEO_S * 1000)
heartbeat_master_port =\
socket.bind_to_random_port("tcp://*")
self.master_heartbeat_address = "{}:{}".format(self.worker_ip,
heartbeat_master_port)
logger.set_dir(
os.path.expanduser('~/.parl_data/worker/{}'.format(
self.master_heartbeat_address.replace(':', '_'))))
self.heartbeat_socket_initialized.set()
logger.info("[Worker] Connect to the master node successfully. "
"({} CPUs)".format(self.cpu_num))
while self.master_is_alive and self.worker_is_alive:
try:
message = socket.recv_multipart()
worker_status = self._get_worker_status()
socket.send_multipart([
remote_constants.HEARTBEAT_TAG,
to_byte(str(worker_status[0])),
to_byte(str(worker_status[1])),
to_byte(worker_status[2]),
to_byte(str(worker_status[3]))
])
except zmq.error.Again as e:
self.master_is_alive = False
except zmq.error.ContextTerminated as e:
break
socket.close(0)
logger.warning(
"[Worker] lost connection with the master, will exit reply heartbeat for master."
)
self.worker_status.clear()
self.log_server_proc.kill()
self.log_server_proc.wait()
# exit the worker
self.worker_is_alive = False
self.exit()
def _reply_heartbeat(self):
"""Reply heartbeat signals to the master node."""
socket = self.ctx.socket(zmq.REP)
socket.linger = 0
socket.setsockopt(zmq.RCVTIMEO,
remote_constants.HEARTBEAT_RCVTIMEO_S * 1000)
reply_master_heartbeat_port =\
socket.bind_to_random_port(addr="tcp://*")
self.reply_master_heartbeat_address = "{}:{}".format(
get_ip_address(), reply_master_heartbeat_port)
self.heartbeat_socket_initialized.set()
connected = False
while self.client_is_alive and self.master_is_alive:
try:
message = socket.recv_multipart()
elapsed_time = datetime.timedelta(seconds=int(time.time() -
self.start_time))
socket.send_multipart([
remote_constants.HEARTBEAT_TAG,
to_byte(self.executable_path),
to_byte(str(self.actor_num)),
to_byte(str(elapsed_time)),
to_byte(str(self.log_monitor_url)),
]) # TODO: remove additional information
except zmq.error.Again as e:
if connected:
logger.warning("[Client] Cannot connect to the master."
"Please check if it is still alive.")
else:
logger.warning(
"[Client] Cannot connect to the master."
"Please check the firewall between client and master.(e.g., ping the master IP)"
)
self.master_is_alive = False
socket.close(0)
logger.warning("Client exit replying heartbeat for master.")
def send_to_zmq_zipped_multi(socket, identity, obj, protocol=2):
"""pickle an object, and zip the pickle before sending it"""
p = pickle.dumps(obj, protocol)
z = zlib.compress(p)
return socket.send_multipart([identity, z], flags=zmq.NOBLOCK)
def process_IN_MODIFY(self, event):
if event.pathname == self.pathname:
socket = zmq.Socket(zmq.Context.instance(), zmq.PUSH)
socket.connect(self.bep)
socket.send_multipart([EMPTY, VERSION, '\x04'])
socket.close()
from machinetalk.protobuf.object_pb2 import ProtocolParameters
# reload(sys)
# sys.setdefaultencoding('utf8')
print "default encoding: " + sys.getdefaultencoding()
context = zmq.Context()
socket = context.socket(zmq.ROUTER)
identity = "worker12"
socket.identity = identity.encode('utf8')
uri = "tcp://machinekit.local:6202"
socket.connect(uri)
cont = Container()
cont.type = MT_PING
txBuffer = cont.SerializeToString()
socket = context.socket(zmq.DEALER)
socket.setsockopt(zmq.RCVTIMEO, 1000)
socket.setsockopt(zmq.LINGER, 10)
print "sending txBuffer: " + txBuffer
socket.send_multipart([identity + txBuffer], zmq.NOBLOCK)
#socket.send(txBuffer)
cont.clear()
def send_message(socket, message):
socket.send_multipart(message)
def send_message(socket, message):
socket.send_multipart(message)
wf = None
for i, port in enumerate(PORTS):
socket = context.socket(zmq.PUB)
socket.bind("tcp://*:%s" % port)
sockets.append(socket)
wf = wave.open(FILE, 'rb')
CHUNK = 4096
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 44100
p = pyaudio.PyAudio()
framecount = 1
while True:
time.sleep(0.001)
framecount += 1
data = wf.readframes(CHUNK)
if len(data) == 0:
wf = wave.open(FILE, 'rb')
data = wf.readframes(CHUNK)
for i, socket in enumerate(sockets):
if len(data) > 0:
socket.send_multipart([TOPIC, data, timenow()])
print "Sending data %d" % framecount
def send_to_zmq_zipped_multi(socket, identity, obj, protocol=2):
"""pickle an object, and zip the pickle before sending it"""
p = pickle.dumps(obj, protocol)
z = zlib.compress(p)
return socket.send_multipart([identity, z], flags=zmq.NOBLOCK)