def tcp_server(self, addrinfo, task=None): task.set_daemon() sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.bind((addrinfo.ip, self.info_port)) except Exception: logger.warning('Could not bind TCP server to %s:%s', addrinfo.ip, self.info_port) raise StopIteration logger.info('dispyadmin TCP server at %s:%s', addrinfo.ip, self.info_port) sock.listen(16) while 1: try: conn, addr = yield sock.accept() except ssl.SSLError as err: logger.debug('SSL connection failed: %s', str(err)) continue except GeneratorExit: break except Exception: logger.debug(traceback.format_exc()) continue Task(self.tcp_req, conn, addr) sock.close()
def __init__(self, cluster, host='', port=8181, poll_sec=10, DocumentRoot=None, keyfile=None, certfile=None): self._cluster_lock = threading.Lock() self._clusters = {} if cluster: cluster_info = self.__class__._ClusterInfo(cluster) self._clusters[cluster.name] = cluster_info if cluster.status_callback is None: cluster.status_callback = functools.partial(self.cluster_status, cluster_info) if not DocumentRoot: DocumentRoot = os.path.join(os.path.dirname(__file__), 'data') if poll_sec < 1: logger.warning('invalid poll_sec value %s; it must be at least 1' % poll_sec) poll_sec = 1 self._poll_sec = poll_sec self._http_handler = None self._server = HTTPServer((host, port), lambda *args: self.__class__._HTTPRequestHandler(self, DocumentRoot, *args)) if certfile: self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile, certfile=certfile, server_side=True) self._httpd_thread = threading.Thread(target=self._server.serve_forever) self._httpd_thread.daemon = True self._httpd_thread.start() logger.info('Started HTTP%s server at %s' % ('s' if certfile else '', str(self._server.socket.getsockname())))
def __init__(self, DocumentRoot, secret='', http_host='localhost', http_port=8181, info_port=51347, node_port=51348, poll_interval=60, ping_interval=600, ip_addrs=[], ipv4_udp_multicast=False, certfile=None, keyfile=None): self.lock = threading.Lock() self.client_uid = None self.client_uid_time = 0 self.nodes = {} self.updates = {} if poll_interval < 1: logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval) poll_interval = 1 self.info_port = info_port self.poll_interval = poll_interval self.ping_interval = ping_interval self.secret = secret self.node_port = node_port self.keyfile = keyfile self.certfile = certfile self.ipv4_udp_multicast = bool(ipv4_udp_multicast) self.addrinfos = {} if not ip_addrs: ip_addrs = [None] for i in range(len(ip_addrs)): ip_addr = ip_addrs[i] addrinfo = dispy.host_addrinfo(host=ip_addr, ipv4_multicast=self.ipv4_udp_multicast) if not addrinfo: logger.warning('Ignoring invalid ip_addr %s', ip_addr) continue self.addrinfos[addrinfo.ip] = addrinfo if not self.addrinfos: raise Exception('No valid IP address found') self.http_port = http_port self.sign = hashlib.sha1(os.urandom(20)) for ip_addr in self.addrinfos: self.sign.update(ip_addr.encode()) self.sign = self.sign.hexdigest() self.auth = dispy.auth_code(self.secret, self.sign) self.tcp_tasks = [] self.udp_tasks = [] udp_addrinfos = {} for addrinfo in self.addrinfos.values(): self.tcp_tasks.append(Task(self.tcp_server, addrinfo)) udp_addrinfos[addrinfo.bind_addr] = addrinfo for bind_addr, addrinfo in udp_addrinfos.items(): self.udp_tasks.append(Task(self.udp_server, addrinfo)) self._server = HTTPServer((http_host, http_port), lambda *args: self.__class__._HTTPRequestHandler(self, DocumentRoot, *args)) if certfile: self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile, certfile=certfile, server_side=True) self.timer = Task(self.timer_proc) self._httpd_thread = threading.Thread(target=self._server.serve_forever) self._httpd_thread.daemon = True self._httpd_thread.start() logger.info('Started HTTP%s server at %s', 's' if certfile else '', ':'.join(map(str, self._server.socket.getsockname())))
def __init__(self, DocumentRoot, secret='', http_host='localhost', poll_interval=60, ping_interval=600, hosts=[], ipv4_udp_multicast=False, certfile=None, keyfile=None): http_port = dispy.config.HTTPServerPort self.node_port = eval(dispy.config.NodePort) self.info_port = eval(dispy.config.ClientPort) self.lock = threading.Lock() self.client_uid = None self.client_uid_time = 0 self.nodes = {} self.updates = {} if poll_interval < 1: logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval) poll_interval = 1 self.poll_interval = poll_interval self.ping_interval = ping_interval self.secret = secret self.keyfile = keyfile self.certfile = certfile self.ipv4_udp_multicast = bool(ipv4_udp_multicast) self.addrinfos = [] if not hosts: hosts = [None] for host in hosts: addrinfo = dispy.host_addrinfo(host=host, ipv4_multicast=self.ipv4_udp_multicast) if not addrinfo: logger.warning('Ignoring invalid host %s', host) continue self.addrinfos.append(addrinfo) if not self.addrinfos: raise Exception('No valid host name / IP address found') self.sign = hashlib.sha1(os.urandom(20)) for addrinfo in self.addrinfos: self.sign.update(addrinfo.ip.encode()) self.sign = self.sign.hexdigest() self.auth = dispy.auth_code(self.secret, self.sign) self.tcp_tasks = [] self.udp_tasks = [] udp_addrinfos = {} for addrinfo in self.addrinfos: self.tcp_tasks.append(Task(self.tcp_server, addrinfo)) udp_addrinfos[addrinfo.bind_addr] = addrinfo for bind_addr, addrinfo in udp_addrinfos.items(): self.udp_tasks.append(Task(self.udp_server, addrinfo)) self._server = HTTPServer((http_host, http_port), lambda *args: self.__class__._HTTPRequestHandler(self, DocumentRoot, *args)) if certfile: self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile, certfile=certfile, server_side=True) self.timer = Task(self.timer_proc) self._httpd_thread = threading.Thread(target=self._server.serve_forever) self._httpd_thread.daemon = True self._httpd_thread.start() self.client_host = self._server.socket.getsockname()[0] logger.info('Started HTTP%s server at %s:%s', 's' if certfile else '', self.client_host, self._server.socket.getsockname()[1])
def shutdown(self, wait=True): """This method should be called by user program to close the http server. """ if wait: logger.info( 'HTTP server waiting for %s seconds for client updates before quitting' % self.poll_interval) time.sleep(self.poll_interval) self._server.shutdown() self._server.server_close()
def shutdown(self, wait=True): """This method should be called by user program to close the http server. """ if wait: logger.info( 'HTTP server waiting for %s seconds for client updates before quitting', self._poll_sec) time.sleep(self._poll_sec) self._server.shutdown() self._server.server_close()
def __init__(self, cluster, host='', port=int(dispy.config.HTTPServerPort), poll_sec=10, DocumentRoot=None, keyfile=None, certfile=None, show_job_args=True): self._cluster_lock = threading.Lock() self._clusters = {} if cluster: cluster_info = self.__class__._ClusterInfo(cluster) self._clusters[cluster.name] = cluster_info http_callback = functools.partial(self.cluster_status, cluster_info) if cluster.status_callback: client_callback = cluster.status_callback def chain_callbacks(status, node, job): http_callback(status, node, job) client_callback(status, node, job) cluster.status_callback = chain_callbacks else: cluster.status_callback = http_callback if not DocumentRoot: DocumentRoot = os.path.join(os.path.dirname(__file__), 'data') if poll_sec < 1: logger.warning('invalid poll_sec value %s; it must be at least 1', poll_sec) poll_sec = 1 self._poll_sec = poll_sec self._show_args = bool(show_job_args) self._server = HTTPServer( (host, int(port)), lambda *args: self.__class__._HTTPRequestHandler( self, DocumentRoot, *args)) if certfile: self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile, certfile=certfile, server_side=True) self._httpd_thread = threading.Thread( target=self._server.serve_forever) self._httpd_thread.daemon = True self._httpd_thread.start() logger.info('Started HTTP%s server at %s', 's' if certfile else '', str(self._server.socket.getsockname()))
else: logger.setLevel(logger.INFO) del config['loglevel'] if config['certfile']: config['certfile'] = os.path.abspath(config['certfile']) else: config['certfile'] = None if config['keyfile']: config['keyfile'] = os.path.abspath(config['keyfile']) else: config['keyfile'] = None args = config.pop('args') logger.info('dispy version %s', _dispy_version) if config['scheduler_node']: cluster = SharedJobCluster(**config) else: del config['scheduler_node'] cluster = JobCluster(**config) jobs = [] for n, arg in enumerate(args, start=1): job = cluster.submit(*(arg.split())) job.id = n jobs.append((job, arg)) for job, args in jobs: job()