def __init__(self, DocumentRoot, secret='', http_host='localhost', http_port=8181,
info_port=51347, node_port=51348, poll_interval=60, ping_interval=600,
ip_addrs=[], ipv4_udp_multicast=False, certfile=None, keyfile=None):
self.lock = threading.Lock()
self.client_uid = None
self.client_uid_time = 0
self.nodes = {}
self.updates = {}
if poll_interval < 1:
logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval)
poll_interval = 1
self.info_port = info_port
self.poll_interval = poll_interval
self.ping_interval = ping_interval
self.secret = secret
self.node_port = node_port
self.keyfile = keyfile
self.certfile = certfile
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
self.addrinfos = {}
if not ip_addrs:
ip_addrs = [None]
for i in range(len(ip_addrs)):
ip_addr = ip_addrs[i]
addrinfo = dispy.host_addrinfo(host=ip_addr, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid ip_addr %s', ip_addr)
continue
self.addrinfos[addrinfo.ip] = addrinfo
if not self.addrinfos:
raise Exception('No valid IP address found')
self.http_port = http_port
self.sign = hashlib.sha1(os.urandom(20))
for ip_addr in self.addrinfos:
self.sign.update(ip_addr.encode())
self.sign = self.sign.hexdigest()
self.auth = dispy.auth_code(self.secret, self.sign)
self.tcp_tasks = []
self.udp_tasks = []
udp_addrinfos = {}
for addrinfo in self.addrinfos.values():
self.tcp_tasks.append(Task(self.tcp_server, addrinfo))
udp_addrinfos[addrinfo.bind_addr] = addrinfo
for bind_addr, addrinfo in udp_addrinfos.items():
self.udp_tasks.append(Task(self.udp_server, addrinfo))
self._server = HTTPServer((http_host, http_port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self.timer = Task(self.timer_proc)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s', 's' if certfile else '',
':'.join(map(str, self._server.socket.getsockname())))
def relay_tcp_proc(self, addrinfo, task=None):
task.set_daemon()
auth_len = len(dispy.auth_code('', ''))
tcp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp_sock.bind((addrinfo.ip, self.relay_port))
tcp_sock.listen(8)
def tcp_req(conn, addr, task=None):
conn.settimeout(dispy.MsgTimeout)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except Exception:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1])
raise StopIteration
finally:
conn.close()
try:
msg = deserialize(msg[len('PING:'.encode()):])
if msg['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
msg['ip_addrs'], msg['version'], __version__)
raise StopIteration
except Exception:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
Task(self.verify_broadcast, addrinfo, msg)
while 1:
conn, addr = yield tcp_sock.accept()
Task(tcp_req, conn, addr)
def relay_tcp_proc(self, addrinfo, task=None):
task.set_daemon()
auth_len = len(dispy.auth_code('', ''))
tcp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp_sock.bind((addrinfo.ip, self.relay_port))
tcp_sock.listen(8)
def tcp_req(conn, addr, task=None):
conn.settimeout(dispy.MsgTimeout)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1])
raise StopIteration
finally:
conn.close()
try:
msg = deserialize(msg[len('PING:'.encode()):])
if msg['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
msg['ip_addrs'], msg['version'], __version__)
raise StopIteration
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
Task(self.verify_broadcast, addrinfo, msg)
while 1:
conn, addr = yield tcp_sock.accept()
Task(tcp_req, conn, addr)
def __init__(self, DocumentRoot, secret='', http_host='localhost',
poll_interval=60, ping_interval=600, hosts=[], ipv4_udp_multicast=False,
certfile=None, keyfile=None):
http_port = dispy.config.HTTPServerPort
self.node_port = eval(dispy.config.NodePort)
self.info_port = eval(dispy.config.ClientPort)
self.lock = threading.Lock()
self.client_uid = None
self.client_uid_time = 0
self.nodes = {}
self.updates = {}
if poll_interval < 1:
logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval)
poll_interval = 1
self.poll_interval = poll_interval
self.ping_interval = ping_interval
self.secret = secret
self.keyfile = keyfile
self.certfile = certfile
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
self.addrinfos = []
if not hosts:
hosts = [None]
for host in hosts:
addrinfo = dispy.host_addrinfo(host=host, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid host %s', host)
continue
self.addrinfos.append(addrinfo)
if not self.addrinfos:
raise Exception('No valid host name / IP address found')
self.sign = hashlib.sha1(os.urandom(20))
for addrinfo in self.addrinfos:
self.sign.update(addrinfo.ip.encode())
self.sign = self.sign.hexdigest()
self.auth = dispy.auth_code(self.secret, self.sign)
self.tcp_tasks = []
self.udp_tasks = []
udp_addrinfos = {}
for addrinfo in self.addrinfos:
self.tcp_tasks.append(Task(self.tcp_server, addrinfo))
udp_addrinfos[addrinfo.bind_addr] = addrinfo
for bind_addr, addrinfo in udp_addrinfos.items():
self.udp_tasks.append(Task(self.udp_server, addrinfo))
self._server = HTTPServer((http_host, http_port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self.timer = Task(self.timer_proc)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
self.client_host = self._server.socket.getsockname()[0]
logger.info('Started HTTP%s server at %s:%s', 's' if certfile else '',
self.client_host, self._server.socket.getsockname()[1])
def listen_tcp_proc(self, coro=None):
coro.set_daemon()
tcp_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_STREAM))
tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp_sock.bind(('', self.listen_port))
tcp_sock.listen(8)
auth_len = len(dispy.auth_code('', ''))
def tcp_task(conn, addr, coro=None):
conn.settimeout(5)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s' %
(addr[0], addr[1]))
raise StopIteration
finally:
conn.close()
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = asyncoro.unserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning(
'Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
raise StopIteration
# TODO: since dispynetrelay is not aware of computations
# closing, if more than one client sends ping, nodes will
# respond to different clients
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
raise StopIteration
logger.debug('relaying ping from %s / %s' %
(info['ip_addrs'], addr[0]))
if self.node_port == self.listen_port:
info[
'relay'] = 'y' # 'check if this message loops back to self
bc_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
yield bc_sock.sendto('PING:'.encode() + asyncoro.serialize(info),
('<broadcast>', self.node_port))
bc_sock.close()
while 1:
conn, addr = yield tcp_sock.accept()
asyncoro.Coro(tcp_task, conn, addr)
def listen_tcp_proc(self, addrinfo, task=None):
task.set_daemon()
tcp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM))
tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp_sock.bind((addrinfo.ip, self.listen_port))
tcp_sock.listen(8)
bc_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
if addrinfo.family == socket.AF_INET:
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
else: # addrinfo.sock_family == socket.AF_INET6
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS,
struct.pack('@i', 1))
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addrinfo.ifn)
bc_sock.bind((addrinfo.ip, 0))
auth_len = len(dispy.auth_code('', ''))
def tcp_req(conn, addr, task=None):
conn.settimeout(5)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1])
raise StopIteration
finally:
conn.close()
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = deserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
raise StopIteration
# TODO: since dispynetrelay is not aware of computations
# closing, if more than one client sends ping, nodes will
# respond to different clients
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
raise StopIteration
logger.debug('relaying ping from %s / %s', info['ip_addrs'], addr[0])
if self.node_port == self.listen_port:
info['relay'] = 'y' # 'check if this message loops back to self
yield bc_sock.sendto('PING:'.encode() + serialize(info),
(self._broadcast, self.node_port))
while 1:
conn, addr = yield tcp_sock.accept()
Task(tcp_req, conn, addr)
def relay_msg(msg, task=None):
relay = {'ip_addrs': self.scheduler_ip_addr, 'port': self.scheduler_port,
'version': __version__}
relay['relay'] = 'y'
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
yield sock.connect((msg['ip_addr'], msg['port']))
yield sock.sendall(dispy.auth_code(self.secret, msg['sign']))
yield sock.send_msg('PING:'.encode() + serialize(relay))
sock.close()
def relay_msg(msg, task=None):
relay = {'ip_addrs': self.scheduler_ip_addr, 'port': self.scheduler_port,
'version': __version__}
relay['relay'] = 'y'
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
yield sock.connect((msg['ip_addr'], msg['port']))
yield sock.sendall(dispy.auth_code(self.secret, msg['sign']))
yield sock.send_msg('PING:'.encode() + serialize(relay))
sock.close()
def verify_broadcast(self, addrinfo, msg, task=None):
if msg.get('relay', None):
raise StopIteration
msg['relay'] = 'y'
# TODO: check if current scheduler is done with nodes?
if msg['sign']:
msg['auth'] = dispy.auth_code(self.secret, msg['sign'])
reply = None
for scheduler_ip_addr in msg['ip_addrs']:
msg['scheduler_ip_addr'] = scheduler_ip_addr
sock = AsyncSocket(socket.socket(addrinfo.family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
try:
yield sock.connect((scheduler_ip_addr, msg['port']))
yield sock.send_msg('RELAY_INFO:'.encode() + serialize(msg))
reply = yield sock.recv_msg()
reply = deserialize(reply)
except:
continue
else:
break
finally:
sock.close()
if not reply:
raise StopIteration
# TODO: since dispynetrelay is not aware of computations closing, if
# more than one client sends ping, nodes will respond to different
# clients
self.scheduler_ip_addr = reply['ip_addrs'] = [scheduler_ip_addr]
self.scheduler_port = reply['port']
bc_sock = AsyncSocket(socket.socket(addrinfo.family,
socket.SOCK_DGRAM))
ttl_bin = struct.pack('@i', 1)
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
bc_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL,
ttl_bin)
else:
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
else: # addrinfo.family == socket.AF_INET6
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS,
ttl_bin)
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
addrinfo.ifn)
bc_sock.bind((addrinfo.ip, 0))
yield bc_sock.sendto('PING:'.encode() + serialize(msg),
(addrinfo.broadcast, self.node_port))
bc_sock.close()
def listen_tcp_proc(self, coro=None):
coro.set_daemon()
tcp_sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp_sock.bind(('', self.listen_port))
tcp_sock.listen(8)
auth_len = len(dispy.auth_code('', ''))
def tcp_task(conn, addr, coro=None):
conn.settimeout(5)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1])
raise StopIteration
finally:
conn.close()
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = asyncoro.unserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning('Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
raise StopIteration
# TODO: since dispynetrelay is not aware of computations
# closing, if more than one client sends ping, nodes will
# respond to different clients
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
raise StopIteration
logger.debug('relaying ping from %s / %s', info['ip_addrs'], addr[0])
if self.node_port == self.listen_port:
info['relay'] = 'y' # 'check if this message loops back to self
bc_sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
yield bc_sock.sendto('PING:'.encode() + asyncoro.serialize(info),
('<broadcast>', self.node_port))
bc_sock.close()
while 1:
conn, addr = yield tcp_sock.accept()
asyncoro.Coro(tcp_task, conn, addr)
def verify_broadcast(self, addrinfo, msg, task=None):
if msg.get('relay', None):
raise StopIteration
msg['relay'] = 'y'
# TODO: check if current scheduler is done with nodes?
if msg['sign']:
msg['auth'] = dispy.auth_code(self.secret, msg['sign'])
reply = None
for scheduler_ip_addr in msg['ip_addrs']:
msg['scheduler_ip_addr'] = scheduler_ip_addr
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(dispy.MsgTimeout)
try:
yield sock.connect((scheduler_ip_addr, msg['port']))
yield sock.send_msg('RELAY_INFO:'.encode() + serialize(msg))
reply = yield sock.recv_msg()
reply = deserialize(reply)
except Exception:
continue
else:
break
finally:
sock.close()
if not reply:
raise StopIteration
# TODO: since dispynetrelay is not aware of computations closing, if
# more than one client sends ping, nodes will respond to different
# clients
self.scheduler_ip_addr = reply['ip_addrs'] = [scheduler_ip_addr]
self.scheduler_port = reply['port']
bc_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
ttl_bin = struct.pack('@i', 1)
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
bc_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin)
else:
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
else: # addrinfo.family == socket.AF_INET6
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, ttl_bin)
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addrinfo.ifn)
bc_sock.bind((addrinfo.ip, 0))
yield bc_sock.sendto('PING:'.encode() + serialize(msg),
(addrinfo.broadcast, self.node_port))
bc_sock.close()
def get_node_info(self, node, task=None):
auth = node._priv.auth
if not auth:
auth = dispy.auth_code(self.secret, node._priv.sign)
sock = AsyncSocket(socket.socket(node._priv.sock_family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(auth)
yield sock.send_msg(b'NODE_INFO:' + serialize({'sign': self.sign}))
info = yield sock.recv_msg()
except Exception:
dispy.logger.debug('Could not get node information from %s:%s',
node.ip_addr, node._priv.port)
# dispy.logger.debug(traceback.format_exc())
raise StopIteration(-1)
finally:
sock.close()
try:
info = deserialize(info)
node.name = info['name']
node.cpus = info['cpus']
node.max_cpus = info['max_cpus']
except Exception:
sign = info.decode()
if node._priv.sign == sign:
node.update_time = time.time()
raise StopIteration(0)
else:
node._priv.sign = sign
ret = yield self.get_node_info(node, task=task)
raise StopIteration(ret)
else:
node._priv.auth = auth
self.set_node_info(node, info)
raise StopIteration(0)
def get_node_info(self, node, task=None):
auth = node._priv.auth
if not auth:
auth = dispy.auth_code(self.secret, node._priv.sign)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(auth)
yield sock.send_msg('NODE_INFO:' + serialize({'sign': self.sign}))
info = yield sock.recv_msg()
except Exception:
dispy.logger.debug('Could not get node information from %s:%s',
node.ip_addr, node._priv.port)
# dispy.logger.debug(traceback.format_exc())
raise StopIteration(-1)
finally:
sock.close()
try:
info = deserialize(info)
node.name = info['name']
node.cpus = info['cpus']
node.max_cpus = info['max_cpus']
except Exception:
sign = info.decode()
if node._priv.sign == sign:
node.update_time = time.time()
raise StopIteration(0)
else:
node._priv.sign = sign
raise StopIteration(yield self.get_node_info(node, task=task))
else:
node._priv.auth = auth
self.set_node_info(node, info)
raise StopIteration(0)
