def serve_clients(self, host, serve, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVE_CLIENTS:'.encode() +
serialize({'serve': serve}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.serve = info['serve']
resp = 0
except Exception:
dispy.logger.debug('Setting serve clients %s to %s failed', host,
serve)
resp = -1
finally:
sock.close()
raise StopIteration(resp)
def service_time(self, host, control, time, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVICE_TIME:'.encode() +
serialize({
'control': control,
'time': time
}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.service_start = info['service_start']
node.service_stop = info['service_stop']
node.service_end = info['service_end']
resp = 0
except Exception:
resp = -1
sock.close()
if resp:
dispy.logger.debug('Setting service %s time of %s to %s failed',
control, host, time)
raise StopIteration(resp)
def tcp_req(conn, addr, task=None):
conn.settimeout(dispy.MsgTimeout)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s',
addr[0], addr[1])
raise StopIteration
finally:
conn.close()
try:
msg = deserialize(msg[len('PING:'.encode()):])
if msg['version'] != __version__:
logger.warning(
'Ignoring %s due to version mismatch: %s / %s',
msg['ip_addrs'], msg['version'], __version__)
raise StopIteration
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
Task(self.verify_broadcast, addrinfo, msg)
def tcp_req(conn, addr, task=None):
conn.settimeout(5)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1])
raise StopIteration
finally:
conn.close()
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = deserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
raise StopIteration
# TODO: since dispynetrelay is not aware of computations
# closing, if more than one client sends ping, nodes will
# respond to different clients
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
raise StopIteration
logger.debug('relaying ping from %s / %s', info['ip_addrs'], addr[0])
if self.node_port == self.listen_port:
info['relay'] = 'y' # 'check if this message loops back to self
yield bc_sock.sendto('PING:'.encode() + serialize(info),
(self._broadcast, self.node_port))
def sched_udp_proc(self, bind_addr, addrinfo, task=None):
task.set_daemon()
def relay_msg(msg, task=None):
relay = {
'ip_addrs': self.scheduler_ip_addr,
'port': self.scheduler_port,
'version': __version__
}
relay['relay'] = 'y'
sock = AsyncSocket(socket.socket(addrinfo.family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
yield sock.connect((msg['ip_addr'], msg['port']))
yield sock.sendall(dispy.auth_code(self.secret, msg['sign']))
yield sock.send_msg('PING:'.encode() + serialize(relay))
sock.close()
sched_sock = AsyncSocket(
socket.socket(addrinfo.family, socket.SOCK_DGRAM))
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
sched_sock.bind((bind_addr, self.scheduler_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(
addrinfo.ip)
sched_sock.setsockopt(socket.IPPROTO_IP,
socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
sched_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP,
mreq)
try:
sched_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY,
1)
except:
pass
while 1:
msg, addr = yield sched_sock.recvfrom(1024)
if not msg.startswith('PING:'.encode()):
logger.debug('Ignoring message from %s (%s)', addr[0], addr[1])
continue
try:
msg = deserialize(msg[len('PING:'.encode()):])
assert msg['version'] == __version__
# assert isinstance(msg['cpus'], int)
except:
continue
if not self.scheduler_ip_addr:
continue
Task(relay_msg, msg)
def listen_udp_proc(self, addrinfo, task=None):
task.set_daemon()
if addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
bc_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
if addrinfo.family == socket.AF_INET:
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
else: # addrinfo.family == socket.AF_INET6
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS,
struct.pack('@i', 1))
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addrinfo.ifn)
bc_sock.bind((addrinfo.ip, 0))
if self.scheduler_ip_addrs and self.scheduler_port:
relay_request = {'ip_addrs': self.scheduler_ip_addrs, 'port': self.scheduler_port,
'version': __version__, 'sign': None}
bc_sock.sendto('PING:'.encode() + serialize(relay_request),
(self._broadcast, self.node_port))
listen_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
if addrinfo.family == socket.AF_INET:
listen_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
else: # addrinfo.family == socket.AF_INET6
listen_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
listen_sock.bind((addrinfo.ip, self.listen_port))
while 1:
msg, addr = yield listen_sock.recvfrom(1024)
if not msg.startswith('PING:'.encode()):
logger.debug('Ignoring message "%s" from %s',
msg[:min(len(msg), 5)], addr[0])
continue
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = deserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
continue
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
continue
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
continue
logger.debug('relaying ping from %s / %s', info['ip_addrs'], addr[0])
if self.node_port == self.listen_port:
info['relay'] = 'y' # 'check if this message loops back to self
yield bc_sock.sendto('PING:'.encode() + serialize(info), addr)
def relay_udp_proc(self, bind_addr, addrinfo, task=None):
task.set_daemon()
relay_sock = AsyncSocket(
socket.socket(addrinfo.family, socket.SOCK_DGRAM))
relay_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
relay_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT,
1)
except Exception:
pass
relay_sock.bind((bind_addr, self.relay_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(
addrinfo.ip)
relay_sock.setsockopt(socket.IPPROTO_IP,
socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
relay_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP,
mreq)
try:
relay_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY,
1)
except Exception:
pass
while 1:
msg, addr = yield relay_sock.recvfrom(1024)
if not msg.startswith('PING:'.encode()):
logger.debug('Ignoring message from %s', addr[0])
continue
if addr[0] in self.ip_addrs:
logger.debug('Ignoring loop back ping from %s' % addr[0])
continue
try:
msg = deserialize(msg[len('PING:'.encode()):])
if msg['version'] != __version__:
logger.warning(
'Ignoring %s due to version mismatch: %s / %s',
msg['ip_addrs'], msg['version'], __version__)
continue
except Exception:
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
logger.debug(traceback.format_exc())
continue
Task(self.verify_broadcast, addrinfo, msg)
def verify_broadcast(self, addrinfo, msg, task=None):
if msg.get('relay', None):
raise StopIteration
msg['relay'] = 'y'
# TODO: check if current scheduler is done with nodes?
if msg['sign']:
msg['auth'] = dispy.auth_code(self.secret, msg['sign'])
reply = None
for scheduler_ip_addr in msg['ip_addrs']:
msg['scheduler_ip_addr'] = scheduler_ip_addr
sock = AsyncSocket(socket.socket(addrinfo.family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
try:
yield sock.connect((scheduler_ip_addr, msg['port']))
yield sock.send_msg('RELAY_INFO:'.encode() + serialize(msg))
reply = yield sock.recv_msg()
reply = deserialize(reply)
except:
continue
else:
break
finally:
sock.close()
if not reply:
raise StopIteration
# TODO: since dispynetrelay is not aware of computations closing, if
# more than one client sends ping, nodes will respond to different
# clients
self.scheduler_ip_addr = reply['ip_addrs'] = [scheduler_ip_addr]
self.scheduler_port = reply['port']
bc_sock = AsyncSocket(socket.socket(addrinfo.family,
socket.SOCK_DGRAM))
ttl_bin = struct.pack('@i', 1)
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
bc_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL,
ttl_bin)
else:
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
else: # addrinfo.family == socket.AF_INET6
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS,
ttl_bin)
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
addrinfo.ifn)
bc_sock.bind((addrinfo.ip, 0))
yield bc_sock.sendto('PING:'.encode() + serialize(msg),
(addrinfo.broadcast, self.node_port))
bc_sock.close()
def sched_udp_proc(self, bind_addr, addrinfo, task=None):
task.set_daemon()
def relay_msg(msg, task=None):
relay = {'ip_addrs': self.scheduler_ip_addr, 'port': self.scheduler_port,
'version': __version__}
relay['relay'] = 'y'
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
yield sock.connect((msg['ip_addr'], msg['port']))
yield sock.sendall(dispy.auth_code(self.secret, msg['sign']))
yield sock.send_msg('PING:'.encode() + serialize(relay))
sock.close()
sched_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
pass
sched_sock.bind((bind_addr, self.scheduler_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(addrinfo.ip)
sched_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
sched_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
try:
sched_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
except Exception:
pass
while 1:
msg, addr = yield sched_sock.recvfrom(1024)
if not msg.startswith('PING:'.encode()):
logger.debug('Ignoring message from %s (%s)', addr[0], addr[1])
continue
try:
msg = deserialize(msg[len('PING:'.encode()):])
assert msg['version'] == __version__
# assert isinstance(msg['cpus'], int)
except Exception:
continue
if not self.scheduler_ip_addr:
continue
Task(relay_msg, msg)
def verify_broadcast(self, addrinfo, msg, task=None):
if msg.get('relay', None):
raise StopIteration
msg['relay'] = 'y'
# TODO: check if current scheduler is done with nodes?
if msg['sign']:
msg['auth'] = dispy.auth_code(self.secret, msg['sign'])
reply = None
for scheduler_ip_addr in msg['ip_addrs']:
msg['scheduler_ip_addr'] = scheduler_ip_addr
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
try:
yield sock.connect((scheduler_ip_addr, msg['port']))
yield sock.send_msg('RELAY_INFO:'.encode() + serialize(msg))
reply = yield sock.recv_msg()
reply = deserialize(reply)
except Exception:
continue
else:
break
finally:
sock.close()
if not reply:
raise StopIteration
# TODO: since dispynetrelay is not aware of computations closing, if
# more than one client sends ping, nodes will respond to different
# clients
self.scheduler_ip_addr = reply['ip_addrs'] = [scheduler_ip_addr]
self.scheduler_port = reply['port']
bc_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
ttl_bin = struct.pack('@i', 1)
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
bc_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin)
else:
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
else: # addrinfo.family == socket.AF_INET6
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, ttl_bin)
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addrinfo.ifn)
bc_sock.bind((addrinfo.ip, 0))
yield bc_sock.sendto('PING:'.encode() + serialize(msg),
(addrinfo.broadcast, self.node_port))
bc_sock.close()
def update_node_info(self, node, task=None):
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg(b'NODE_STATUS:')
info = yield sock.recv_msg()
info = deserialize(info)
if isinstance(info, dict):
self.set_node_info(node, info)
except Exception:
# TODO: remove node if update is long ago?
pass
finally:
sock.close()
def update_node_info(self, node, task=None):
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('NODE_STATUS:')
info = yield sock.recv_msg()
info = deserialize(info)
if isinstance(info, dict):
self.set_node_info(node, info)
except Exception:
logger.debug('Could not update node at %s:%s', node.ip_addr, node._priv.port)
# TODO: remove node if update is long ago?
finally:
sock.close()
def relay_udp_proc(self, bind_addr, addrinfo, task=None):
task.set_daemon()
relay_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
relay_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
relay_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
pass
relay_sock.bind((bind_addr, self.relay_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(addrinfo.ip)
relay_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
relay_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
try:
relay_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
except Exception:
pass
while 1:
msg, addr = yield relay_sock.recvfrom(1024)
if not msg.startswith('PING:'.encode()):
logger.debug('Ignoring message from %s', addr[0])
continue
if addr[0] in self.ip_addrs:
logger.debug('Ignoring loop back ping from %s' % addr[0])
continue
try:
msg = deserialize(msg[len('PING:'.encode()):])
if msg['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
msg['ip_addrs'], msg['version'], __version__)
continue
except Exception:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
continue
Task(self.verify_broadcast, addrinfo, msg)
def sched_udp_proc(self, addrinfo, task=None):
task.set_daemon()
sched_sock = AsyncSocket(
socket.socket(addrinfo.family, socket.SOCK_DGRAM))
if addrinfo.family == socket.AF_INET:
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
else: # addrinfo.family == socket.AF_INET6
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
sched_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP,
mreq)
sched_sock.bind((addrinfo.ip, self.scheduler_port))
while 1:
msg, addr = yield sched_sock.recvfrom(1024)
if (not msg.startswith('PING:'.encode())
or not self.scheduler_ip_addrs or not self.scheduler_port):
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
continue
try:
info = deserialize(msg[len('PING:'.encode()):])
assert info['version'] == __version__
# assert isinstance(info['cpus'], int)
except:
logger.debug(traceback.format_exc())
msg = {
'ip_addrs': self.scheduler_ip_addrs,
'port': self.scheduler_port,
'version': __version__
}
if info.get('relay', None):
logger.debug('Ignoring ping back from %s: %s', addr[0], info)
continue
msg['relay'] = 'y'
relay_sock = AsyncSocket(
socket.socket(addrinfo.family, socket.SOCK_DGRAM))
relay_sock.bind((addrinfo.ip, 0))
yield relay_sock.sendto('PING:'.encode() + serialize(msg),
(info['ip_addr'], info['port']))
relay_sock.close()
def set_cpus(self, host, cpus, task=None):
node = self.nodes.get(host, None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SET_CPUS:' + serialize({'cpus': cpus}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.cpus = info['cpus']
except Exception:
dispy.logger.debug('Setting cpus of %s to %s failed', host, cpus)
raise StopIteration(-1)
else:
raise StopIteration(0)
finally:
sock.close()
def serve_clients(self, host, serve, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVE_CLIENTS:' + serialize({'serve': serve}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.serve = info['serve']
resp = 0
except Exception:
dispy.logger.debug('Setting serve %s to %s failed', host, serve)
resp = -1
finally:
sock.close()
raise StopIteration(resp)
def set_cpus(self, host, cpus, task=None):
node = self.nodes.get(host, None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SET_CPUS:'.encode() + serialize({'cpus': cpus}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.cpus = info['cpus']
except Exception:
dispy.logger.debug('Setting cpus of %s to %s failed', host, cpus)
raise StopIteration(-1)
else:
raise StopIteration(0)
finally:
sock.close()
def tcp_req(self, conn, addr, task=None):
conn.settimeout(MsgTimeout)
msg = yield conn.recv_msg()
if msg.startswith(b'NODE_INFO:'):
try:
info = deserialize(msg[len(b'NODE_INFO:'):])
dispy.logger.info('info: %s', info)
node = info.get('ip_addr', None)
if info.get('version', None) != _dispy_version:
dispy.logger.warning('Ignoring node at %s due to version mismatch (%s != %s)',
info.get('ip_addr', None),
info.get('version', None), _dispy_version)
raise StopIteration
assert info['sign']
info['family'] = conn.family
except Exception:
# dispy.logger.debug(traceback.format_exc())
raise StopIteration
finally:
conn.close()
yield self.add_node(info)
raise StopIteration
def tcp_req(self, conn, addr, task=None):
conn.settimeout(MsgTimeout)
msg = yield conn.recv_msg()
if msg.startswith('NODE_INFO:'):
try:
info = deserialize(msg[len('NODE_INFO:'):])
dispy.logger.info('info: %s', info)
node = info.get('ip_addr', None)
if info.get('version', None) != _dispy_version:
dispy.logger.warning('Ignoring node at %s due to version mismatch (%s != %s)',
info.get('ip_addr', None),
info.get('version', None), _dispy_version)
raise StopIteration
assert info['sign']
info['family'] = conn.family
except Exception:
# dispy.logger.debug(traceback.format_exc())
raise StopIteration
finally:
conn.close()
yield self.add_node(info)
raise StopIteration
def get_node_info(self, node, task=None):
auth = node._priv.auth
if not auth:
auth = dispy.auth_code(self.secret, node._priv.sign)
sock = AsyncSocket(socket.socket(node._priv.sock_family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(auth)
yield sock.send_msg(b'NODE_INFO:' + serialize({'sign': self.sign}))
info = yield sock.recv_msg()
except Exception:
dispy.logger.debug('Could not get node information from %s:%s',
node.ip_addr, node._priv.port)
# dispy.logger.debug(traceback.format_exc())
raise StopIteration(-1)
finally:
sock.close()
try:
info = deserialize(info)
node.name = info['name']
node.cpus = info['cpus']
node.max_cpus = info['max_cpus']
except Exception:
sign = info.decode()
if node._priv.sign == sign:
node.update_time = time.time()
raise StopIteration(0)
else:
node._priv.sign = sign
ret = yield self.get_node_info(node, task=task)
raise StopIteration(ret)
else:
node._priv.auth = auth
self.set_node_info(node, info)
raise StopIteration(0)
def tcp_req(conn, addr, task=None):
conn.settimeout(dispy.MsgTimeout)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except Exception:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1])
raise StopIteration
finally:
conn.close()
try:
msg = deserialize(msg[len('PING:'.encode()):])
if msg['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
msg['ip_addrs'], msg['version'], __version__)
raise StopIteration
except Exception:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
Task(self.verify_broadcast, addrinfo, msg)
def service_time(self, host, control, time, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVICE_TIME:' + serialize({'control': control, 'time': time}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.service_start = info['service_start']
node.service_stop = info['service_stop']
node.service_end = info['service_end']
resp = 0
except Exception:
resp = -1
sock.close()
if resp:
dispy.logger.debug('Setting service %s time of %s to %s failed', control, host, time)
raise StopIteration(resp)
def get_node_info(self, node, task=None):
auth = node._priv.auth
if not auth:
auth = dispy.auth_code(self.secret, node._priv.sign)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(auth)
yield sock.send_msg('NODE_INFO:' + serialize({'sign': self.sign}))
info = yield sock.recv_msg()
except Exception:
dispy.logger.debug('Could not get node information from %s:%s',
node.ip_addr, node._priv.port)
# dispy.logger.debug(traceback.format_exc())
raise StopIteration(-1)
finally:
sock.close()
try:
info = deserialize(info)
node.name = info['name']
node.cpus = info['cpus']
node.max_cpus = info['max_cpus']
except Exception:
sign = info.decode()
if node._priv.sign == sign:
node.update_time = time.time()
raise StopIteration(0)
else:
node._priv.sign = sign
raise StopIteration(yield self.get_node_info(node, task=task))
else:
node._priv.auth = auth
self.set_node_info(node, info)
raise StopIteration(0)
def udp_server(self, addrinfo, task=None):
task.set_daemon()
udp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
pass
udp_sock.bind((addrinfo.bind_addr, self.info_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(addrinfo.ip)
udp_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
try:
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
except Exception:
pass
while 1:
msg, addr = yield udp_sock.recvfrom(1000)
if msg.startswith('PING:'):
try:
info = deserialize(msg[len('PING:'):])
if info['version'] != _dispy_version:
logger.warning('Ignoring %s due to version mismatch', addr[0])
continue
assert info['port'] > 0
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node:
if node._priv.sign == info['sign']:
Task(self.update_node_info, node)
else:
node._priv.sign = info['sign']
node._priv.auth = None
Task(self.get_node_info, node)
else:
info['family'] = addrinfo.family
Task(self.add_node, info)
elif msg.startswith('TERMINATED:'):
try:
info = deserialize(msg[len('TERMINATED:'):])
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node and node._priv.sign == info['sign']:
with self.lock:
self.nodes.pop(info['ip_addr'], None)
def udp_server(self, addrinfo, task=None):
task.set_daemon()
udp_sock = AsyncSocket(
socket.socket(addrinfo.family, socket.SOCK_DGRAM))
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
pass
udp_sock.bind((addrinfo.bind_addr, self.info_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(
addrinfo.ip)
udp_sock.setsockopt(socket.IPPROTO_IP,
socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP,
mreq)
try:
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
except Exception:
pass
while 1:
msg, addr = yield udp_sock.recvfrom(1000)
if msg.startswith(b'PING:'):
try:
info = deserialize(msg[len(b'PING:'):])
if info['version'] != _dispy_version:
logger.warning('Ignoring %s due to version mismatch',
addr[0])
continue
assert info['port'] > 0
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node:
if node._priv.sign == info['sign']:
Task(self.update_node_info, node)
else:
node._priv.sign = info['sign']
node._priv.auth = None
Task(self.get_node_info, node)
else:
info['family'] = addrinfo.family
Task(self.add_node, info)
elif msg.startswith(b'TERMINATED:'):
try:
info = deserialize(msg[len(b'TERMINATED:'):])
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node and node._priv.sign == info['sign']:
with self.lock:
self.nodes.pop(info['ip_addr'], None)
