def __init__(self, DocumentRoot, secret='', http_host='localhost', http_port=8181, info_port=51347, node_port=51348, poll_interval=60, ping_interval=600, ip_addrs=[], ipv4_udp_multicast=False, certfile=None, keyfile=None): self.lock = threading.Lock() self.client_uid = None self.client_uid_time = 0 self.nodes = {} self.updates = {} if poll_interval < 1: logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval) poll_interval = 1 self.info_port = info_port self.poll_interval = poll_interval self.ping_interval = ping_interval self.secret = secret self.node_port = node_port self.keyfile = keyfile self.certfile = certfile self.ipv4_udp_multicast = bool(ipv4_udp_multicast) self.addrinfos = {} if not ip_addrs: ip_addrs = [None] for i in range(len(ip_addrs)): ip_addr = ip_addrs[i] addrinfo = dispy.host_addrinfo(host=ip_addr, ipv4_multicast=self.ipv4_udp_multicast) if not addrinfo: logger.warning('Ignoring invalid ip_addr %s', ip_addr) continue self.addrinfos[addrinfo.ip] = addrinfo if not self.addrinfos: raise Exception('No valid IP address found') self.http_port = http_port self.sign = hashlib.sha1(os.urandom(20)) for ip_addr in self.addrinfos: self.sign.update(ip_addr.encode()) self.sign = self.sign.hexdigest() self.auth = dispy.auth_code(self.secret, self.sign) self.tcp_tasks = [] self.udp_tasks = [] udp_addrinfos = {} for addrinfo in self.addrinfos.values(): self.tcp_tasks.append(Task(self.tcp_server, addrinfo)) udp_addrinfos[addrinfo.bind_addr] = addrinfo for bind_addr, addrinfo in udp_addrinfos.items(): self.udp_tasks.append(Task(self.udp_server, addrinfo)) self._server = HTTPServer((http_host, http_port), lambda *args: self.__class__._HTTPRequestHandler(self, DocumentRoot, *args)) if certfile: self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile, certfile=certfile, server_side=True) self.timer = Task(self.timer_proc) self._httpd_thread = threading.Thread(target=self._server.serve_forever) self._httpd_thread.daemon = True self._httpd_thread.start() logger.info('Started HTTP%s server at %s', 's' if certfile else '', ':'.join(map(str, self._server.socket.getsockname())))
def relay_tcp_proc(self, addrinfo, task=None): task.set_daemon() auth_len = len(dispy.auth_code('', '')) tcp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcp_sock.bind((addrinfo.ip, self.relay_port)) tcp_sock.listen(8) def tcp_req(conn, addr, task=None): conn.settimeout(dispy.MsgTimeout) try: msg = yield conn.recvall(auth_len) msg = yield conn.recv_msg() except Exception: logger.debug(traceback.format_exc()) logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1]) raise StopIteration finally: conn.close() try: msg = deserialize(msg[len('PING:'.encode()):]) if msg['version'] != __version__: logger.warning('Ignoring %s due to version mismatch: %s / %s', msg['ip_addrs'], msg['version'], __version__) raise StopIteration except Exception: logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1]) logger.debug(traceback.format_exc()) raise StopIteration Task(self.verify_broadcast, addrinfo, msg) while 1: conn, addr = yield tcp_sock.accept() Task(tcp_req, conn, addr)
def relay_tcp_proc(self, addrinfo, task=None): task.set_daemon() auth_len = len(dispy.auth_code('', '')) tcp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcp_sock.bind((addrinfo.ip, self.relay_port)) tcp_sock.listen(8) def tcp_req(conn, addr, task=None): conn.settimeout(dispy.MsgTimeout) try: msg = yield conn.recvall(auth_len) msg = yield conn.recv_msg() except: logger.debug(traceback.format_exc()) logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1]) raise StopIteration finally: conn.close() try: msg = deserialize(msg[len('PING:'.encode()):]) if msg['version'] != __version__: logger.warning('Ignoring %s due to version mismatch: %s / %s', msg['ip_addrs'], msg['version'], __version__) raise StopIteration except: logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1]) logger.debug(traceback.format_exc()) raise StopIteration Task(self.verify_broadcast, addrinfo, msg) while 1: conn, addr = yield tcp_sock.accept() Task(tcp_req, conn, addr)
def __init__(self, DocumentRoot, secret='', http_host='localhost', poll_interval=60, ping_interval=600, hosts=[], ipv4_udp_multicast=False, certfile=None, keyfile=None): http_port = dispy.config.HTTPServerPort self.node_port = eval(dispy.config.NodePort) self.info_port = eval(dispy.config.ClientPort) self.lock = threading.Lock() self.client_uid = None self.client_uid_time = 0 self.nodes = {} self.updates = {} if poll_interval < 1: logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval) poll_interval = 1 self.poll_interval = poll_interval self.ping_interval = ping_interval self.secret = secret self.keyfile = keyfile self.certfile = certfile self.ipv4_udp_multicast = bool(ipv4_udp_multicast) self.addrinfos = [] if not hosts: hosts = [None] for host in hosts: addrinfo = dispy.host_addrinfo(host=host, ipv4_multicast=self.ipv4_udp_multicast) if not addrinfo: logger.warning('Ignoring invalid host %s', host) continue self.addrinfos.append(addrinfo) if not self.addrinfos: raise Exception('No valid host name / IP address found') self.sign = hashlib.sha1(os.urandom(20)) for addrinfo in self.addrinfos: self.sign.update(addrinfo.ip.encode()) self.sign = self.sign.hexdigest() self.auth = dispy.auth_code(self.secret, self.sign) self.tcp_tasks = [] self.udp_tasks = [] udp_addrinfos = {} for addrinfo in self.addrinfos: self.tcp_tasks.append(Task(self.tcp_server, addrinfo)) udp_addrinfos[addrinfo.bind_addr] = addrinfo for bind_addr, addrinfo in udp_addrinfos.items(): self.udp_tasks.append(Task(self.udp_server, addrinfo)) self._server = HTTPServer((http_host, http_port), lambda *args: self.__class__._HTTPRequestHandler(self, DocumentRoot, *args)) if certfile: self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile, certfile=certfile, server_side=True) self.timer = Task(self.timer_proc) self._httpd_thread = threading.Thread(target=self._server.serve_forever) self._httpd_thread.daemon = True self._httpd_thread.start() self.client_host = self._server.socket.getsockname()[0] logger.info('Started HTTP%s server at %s:%s', 's' if certfile else '', self.client_host, self._server.socket.getsockname()[1])
def listen_tcp_proc(self, coro=None): coro.set_daemon() tcp_sock = asyncoro.AsyncSocket( socket.socket(socket.AF_INET, socket.SOCK_STREAM)) tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcp_sock.bind(('', self.listen_port)) tcp_sock.listen(8) auth_len = len(dispy.auth_code('', '')) def tcp_task(conn, addr, coro=None): conn.settimeout(5) try: msg = yield conn.recvall(auth_len) msg = yield conn.recv_msg() except: logger.debug(traceback.format_exc()) logger.debug('Ignoring invalid TCP message from %s:%s' % (addr[0], addr[1])) raise StopIteration finally: conn.close() logger.debug('Ping message from %s (%s)', addr[0], addr[1]) try: info = asyncoro.unserialize(msg[len('PING:'.encode()):]) if info['version'] != __version__: logger.warning( 'Ignoring %s due to version mismatch: %s / %s', info['ip_addrs'], info['version'], __version__) raise StopIteration # TODO: since dispynetrelay is not aware of computations # closing, if more than one client sends ping, nodes will # respond to different clients self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]] self.scheduler_port = info['port'] except: logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1]) logger.debug(traceback.format_exc()) raise StopIteration if info.get('relay', None): logger.debug('Ignoring ping back (from %s)', addr[0]) raise StopIteration logger.debug('relaying ping from %s / %s' % (info['ip_addrs'], addr[0])) if self.node_port == self.listen_port: info[ 'relay'] = 'y' # 'check if this message loops back to self bc_sock = asyncoro.AsyncSocket( socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) yield bc_sock.sendto('PING:'.encode() + asyncoro.serialize(info), ('<broadcast>', self.node_port)) bc_sock.close() while 1: conn, addr = yield tcp_sock.accept() asyncoro.Coro(tcp_task, conn, addr)
def listen_tcp_proc(self, addrinfo, task=None): task.set_daemon() tcp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM)) tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcp_sock.bind((addrinfo.ip, self.listen_port)) tcp_sock.listen(8) bc_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM)) if addrinfo.family == socket.AF_INET: bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) else: # addrinfo.sock_family == socket.AF_INET6 bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, struct.pack('@i', 1)) bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addrinfo.ifn) bc_sock.bind((addrinfo.ip, 0)) auth_len = len(dispy.auth_code('', '')) def tcp_req(conn, addr, task=None): conn.settimeout(5) try: msg = yield conn.recvall(auth_len) msg = yield conn.recv_msg() except: logger.debug(traceback.format_exc()) logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1]) raise StopIteration finally: conn.close() logger.debug('Ping message from %s (%s)', addr[0], addr[1]) try: info = deserialize(msg[len('PING:'.encode()):]) if info['version'] != __version__: logger.warning('Ignoring %s due to version mismatch: %s / %s', info['ip_addrs'], info['version'], __version__) raise StopIteration # TODO: since dispynetrelay is not aware of computations # closing, if more than one client sends ping, nodes will # respond to different clients self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]] self.scheduler_port = info['port'] except: logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1]) logger.debug(traceback.format_exc()) raise StopIteration if info.get('relay', None): logger.debug('Ignoring ping back (from %s)', addr[0]) raise StopIteration logger.debug('relaying ping from %s / %s', info['ip_addrs'], addr[0]) if self.node_port == self.listen_port: info['relay'] = 'y' # 'check if this message loops back to self yield bc_sock.sendto('PING:'.encode() + serialize(info), (self._broadcast, self.node_port)) while 1: conn, addr = yield tcp_sock.accept() Task(tcp_req, conn, addr)
def relay_msg(msg, task=None): relay = {'ip_addrs': self.scheduler_ip_addr, 'port': self.scheduler_port, 'version': __version__} relay['relay'] = 'y' sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.settimeout(dispy.MsgTimeout) yield sock.connect((msg['ip_addr'], msg['port'])) yield sock.sendall(dispy.auth_code(self.secret, msg['sign'])) yield sock.send_msg('PING:'.encode() + serialize(relay)) sock.close()
def verify_broadcast(self, addrinfo, msg, task=None): if msg.get('relay', None): raise StopIteration msg['relay'] = 'y' # TODO: check if current scheduler is done with nodes? if msg['sign']: msg['auth'] = dispy.auth_code(self.secret, msg['sign']) reply = None for scheduler_ip_addr in msg['ip_addrs']: msg['scheduler_ip_addr'] = scheduler_ip_addr sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.settimeout(dispy.MsgTimeout) try: yield sock.connect((scheduler_ip_addr, msg['port'])) yield sock.send_msg('RELAY_INFO:'.encode() + serialize(msg)) reply = yield sock.recv_msg() reply = deserialize(reply) except: continue else: break finally: sock.close() if not reply: raise StopIteration # TODO: since dispynetrelay is not aware of computations closing, if # more than one client sends ping, nodes will respond to different # clients self.scheduler_ip_addr = reply['ip_addrs'] = [scheduler_ip_addr] self.scheduler_port = reply['port'] bc_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM)) ttl_bin = struct.pack('@i', 1) if addrinfo.family == socket.AF_INET: if self.ipv4_udp_multicast: bc_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin) else: bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) else: # addrinfo.family == socket.AF_INET6 bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, ttl_bin) bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addrinfo.ifn) bc_sock.bind((addrinfo.ip, 0)) yield bc_sock.sendto('PING:'.encode() + serialize(msg), (addrinfo.broadcast, self.node_port)) bc_sock.close()
def listen_tcp_proc(self, coro=None): coro.set_daemon() tcp_sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcp_sock.bind(('', self.listen_port)) tcp_sock.listen(8) auth_len = len(dispy.auth_code('', '')) def tcp_task(conn, addr, coro=None): conn.settimeout(5) try: msg = yield conn.recvall(auth_len) msg = yield conn.recv_msg() except: logger.debug(traceback.format_exc()) logger.debug('Ignoring invalid TCP message from %s:%s', addr[0], addr[1]) raise StopIteration finally: conn.close() logger.debug('Ping message from %s (%s)', addr[0], addr[1]) try: info = asyncoro.unserialize(msg[len('PING:'.encode()):]) if info['version'] != __version__: logger.warning('Ignoring %s due to version mismatch: %s / %s', info['ip_addrs'], info['version'], __version__) raise StopIteration # TODO: since dispynetrelay is not aware of computations # closing, if more than one client sends ping, nodes will # respond to different clients self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]] self.scheduler_port = info['port'] except: logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1]) logger.debug(traceback.format_exc()) raise StopIteration if info.get('relay', None): logger.debug('Ignoring ping back (from %s)', addr[0]) raise StopIteration logger.debug('relaying ping from %s / %s', info['ip_addrs'], addr[0]) if self.node_port == self.listen_port: info['relay'] = 'y' # 'check if this message loops back to self bc_sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) yield bc_sock.sendto('PING:'.encode() + asyncoro.serialize(info), ('<broadcast>', self.node_port)) bc_sock.close() while 1: conn, addr = yield tcp_sock.accept() asyncoro.Coro(tcp_task, conn, addr)
def verify_broadcast(self, addrinfo, msg, task=None): if msg.get('relay', None): raise StopIteration msg['relay'] = 'y' # TODO: check if current scheduler is done with nodes? if msg['sign']: msg['auth'] = dispy.auth_code(self.secret, msg['sign']) reply = None for scheduler_ip_addr in msg['ip_addrs']: msg['scheduler_ip_addr'] = scheduler_ip_addr sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.settimeout(dispy.MsgTimeout) try: yield sock.connect((scheduler_ip_addr, msg['port'])) yield sock.send_msg('RELAY_INFO:'.encode() + serialize(msg)) reply = yield sock.recv_msg() reply = deserialize(reply) except Exception: continue else: break finally: sock.close() if not reply: raise StopIteration # TODO: since dispynetrelay is not aware of computations closing, if # more than one client sends ping, nodes will respond to different # clients self.scheduler_ip_addr = reply['ip_addrs'] = [scheduler_ip_addr] self.scheduler_port = reply['port'] bc_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM)) ttl_bin = struct.pack('@i', 1) if addrinfo.family == socket.AF_INET: if self.ipv4_udp_multicast: bc_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin) else: bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) else: # addrinfo.family == socket.AF_INET6 bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, ttl_bin) bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addrinfo.ifn) bc_sock.bind((addrinfo.ip, 0)) yield bc_sock.sendto('PING:'.encode() + serialize(msg), (addrinfo.broadcast, self.node_port)) bc_sock.close()
def get_node_info(self, node, task=None): auth = node._priv.auth if not auth: auth = dispy.auth_code(self.secret, node._priv.sign) sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.settimeout(MsgTimeout) try: yield sock.connect((node.ip_addr, node._priv.port)) yield sock.sendall(auth) yield sock.send_msg(b'NODE_INFO:' + serialize({'sign': self.sign})) info = yield sock.recv_msg() except Exception: dispy.logger.debug('Could not get node information from %s:%s', node.ip_addr, node._priv.port) # dispy.logger.debug(traceback.format_exc()) raise StopIteration(-1) finally: sock.close() try: info = deserialize(info) node.name = info['name'] node.cpus = info['cpus'] node.max_cpus = info['max_cpus'] except Exception: sign = info.decode() if node._priv.sign == sign: node.update_time = time.time() raise StopIteration(0) else: node._priv.sign = sign ret = yield self.get_node_info(node, task=task) raise StopIteration(ret) else: node._priv.auth = auth self.set_node_info(node, info) raise StopIteration(0)
def get_node_info(self, node, task=None): auth = node._priv.auth if not auth: auth = dispy.auth_code(self.secret, node._priv.sign) sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.settimeout(MsgTimeout) try: yield sock.connect((node.ip_addr, node._priv.port)) yield sock.sendall(auth) yield sock.send_msg('NODE_INFO:' + serialize({'sign': self.sign})) info = yield sock.recv_msg() except Exception: dispy.logger.debug('Could not get node information from %s:%s', node.ip_addr, node._priv.port) # dispy.logger.debug(traceback.format_exc()) raise StopIteration(-1) finally: sock.close() try: info = deserialize(info) node.name = info['name'] node.cpus = info['cpus'] node.max_cpus = info['max_cpus'] except Exception: sign = info.decode() if node._priv.sign == sign: node.update_time = time.time() raise StopIteration(0) else: node._priv.sign = sign raise StopIteration(yield self.get_node_info(node, task=task)) else: node._priv.auth = auth self.set_node_info(node, info) raise StopIteration(0)