def tcp_server(self, addrinfo, task=None): task.set_daemon() sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.bind((addrinfo.ip, self.info_port)) except Exception: logger.warning('Could not bind TCP server to %s:%s', addrinfo.ip, self.info_port) raise StopIteration logger.info('dispyadmin TCP server at %s:%s', addrinfo.ip, self.info_port) sock.listen(16) while 1: try: conn, addr = yield sock.accept() except ssl.SSLError as err: logger.debug('SSL connection failed: %s', str(err)) continue except GeneratorExit: break except Exception: logger.debug(traceback.format_exc()) continue Task(self.tcp_req, conn, addr) sock.close()
def do_GET(self): path = urlparse(self.path).path.lstrip('/') if path == '' or path == 'index.html': path = 'admin.html' path = os.path.join(self.DocumentRoot, path) try: with open(path) as fd: data = fd.read() if path.endswith('.html'): if path.endswith('admin.html') or path.endswith( 'admin_node.html'): data = data % { 'POLL_INTERVAL': str(self._ctx.poll_interval), 'NODE_PORT': str(self._ctx.node_port) } content_type = 'text/html' elif path.endswith('.js'): content_type = 'text/javascript' elif path.endswith('.css'): content_type = 'text/css' elif path.endswith('.ico'): content_type = 'image/x-icon' self.send_response(200) self.send_header('Content-Type', content_type) if content_type == 'text/css' or content_type == 'text/javascript': self.send_header('Cache-Control', 'private, max-age=86400') self.end_headers() self.wfile.write(data.encode()) return except Exception: logger.warning('HTTP client %s: Could not read/send "%s"', self.client_address[0], path) logger.debug(traceback.format_exc()) self.send_error(404) return
def do_GET(self): parsed_path = urlparse(self.path) path = parsed_path.path.lstrip('/') if path == '' or path == 'index.html': path = 'admin.html' path = os.path.join(self.DocumentRoot, path) try: with open(path) as fd: data = fd.read() if path.endswith('.html'): if path.endswith('admin.html') or path.endswith('admin_node.html'): data = data % {'POLL_INTERVAL': str(self._ctx.poll_interval), 'NODE_PORT': str(self._ctx.node_port)} content_type = 'text/html' elif path.endswith('.js'): content_type = 'text/javascript' elif path.endswith('.css'): content_type = 'text/css' elif path.endswith('.ico'): content_type = 'image/x-icon' self.send_response(200) self.send_header('Content-Type', content_type) if content_type == 'text/css' or content_type == 'text/javascript': # self.send_header('Cache-Control', 'private, max-age=86400') self.send_header('Cache-Control', 'private, max-age=30') self.end_headers() self.wfile.write(data.encode()) return except Exception: logger.warning('HTTP client %s: Could not read/send "%s"', self.client_address[0], path) logger.debug(traceback.format_exc()) self.send_error(404) return
def update_node_info(self, node, task=None): sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.settimeout(MsgTimeout) try: yield sock.connect((node.ip_addr, node._priv.port)) yield sock.sendall(node._priv.auth) yield sock.send_msg('NODE_STATUS:') info = yield sock.recv_msg() info = deserialize(info) if isinstance(info, dict): self.set_node_info(node, info) except Exception: logger.debug('Could not update node at %s:%s', node.ip_addr, node._priv.port) # TODO: remove node if update is long ago? finally: sock.close()
def tcp_server(self, addrinfo, task=None): task.set_daemon() sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM), keyfile=self.keyfile, certfile=self.certfile) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.bind((addrinfo.ip, self.info_port)) except Exception: logger.warning('Could not bind TCP server to %s:%s', addrinfo.ip, self.info_port) raise StopIteration logger.debug('dispyadmin TCP server at %s:%s', addrinfo.ip, self.info_port) sock.listen(16) while 1: try: conn, addr = yield sock.accept() except ssl.SSLError as err: logger.debug('SSL connection failed: %s', str(err)) continue except GeneratorExit: break except Exception: logger.debug(traceback.format_exc()) continue Task(self.tcp_req, conn, addr) sock.close()
def udp_server(self, addrinfo, task=None): task.set_daemon() udp_sock = AsyncSocket( socket.socket(addrinfo.family, socket.SOCK_DGRAM)) udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): try: udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) except Exception: pass udp_sock.bind((addrinfo.bind_addr, self.info_port)) if addrinfo.family == socket.AF_INET: if self.ipv4_udp_multicast: mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton( addrinfo.ip) udp_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) else: # addrinfo.family == socket.AF_INET6: mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast) mreq += struct.pack('@I', addrinfo.ifn) udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq) try: udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) except Exception: pass while 1: msg, addr = yield udp_sock.recvfrom(1000) if msg.startswith(b'PING:'): try: info = deserialize(msg[len(b'PING:'):]) if info['version'] != _dispy_version: logger.warning('Ignoring %s due to version mismatch', addr[0]) continue assert info['port'] > 0 assert info['ip_addr'] except Exception: logger.debug('Ignoring node %s', addr[0]) continue node = self.nodes.get(info['ip_addr'], None) if node: if node._priv.sign == info['sign']: Task(self.update_node_info, node) else: node._priv.sign = info['sign'] node._priv.auth = None Task(self.get_node_info, node) else: info['family'] = addrinfo.family Task(self.add_node, info) elif msg.startswith(b'TERMINATED:'): try: info = deserialize(msg[len(b'TERMINATED:'):]) assert info['ip_addr'] except Exception: logger.debug('Ignoring node %s', addr[0]) continue node = self.nodes.get(info['ip_addr'], None) if node and node._priv.sign == info['sign']: with self.lock: self.nodes.pop(info['ip_addr'], None)
def do_POST(self): try: form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}) client_request = self.path[1:] except Exception: logger.debug('Ignoring invalid POST request from %s', self.client_address[0]) self.send_error(400) return if client_request == 'update': uid = None for item in form.list: if item.name == 'uid': uid = item.value.strip() break if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() self._ctx.lock.acquire() nodes = self.__class__.json_encode_nodes(self._ctx.updates) self._ctx.updates.clear() self._ctx.lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return elif client_request == 'node_info': ip_addr = None uid = None for item in form.list: if item.name == 'host': # if it looks like IP address, skip resolving if re.match(DispyAdminServer._NodeInfo.ip_re, item.value): ip_addr = item.value else: ip_addr = dispy._node_ipaddr(item.value) elif item.name == 'uid': uid = item.value.strip() if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() node = self._ctx.nodes.get(ip_addr, None) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() if node: node = dict(node.__dict__) node.pop('_priv', None) if node['avail_info']: node['avail_info'] = node['avail_info'].__dict__ else: node = {} self.wfile.write(json.dumps(node).encode()) return elif client_request == 'status': uid = None for item in form.list: if item.name == 'uid': uid = item.value.strip() break if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() self._ctx.lock.acquire() nodes = self.__class__.json_encode_nodes(self._ctx.nodes) self._ctx.lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return elif client_request == 'get_uid': uid = None for item in form.list: if item.name == 'uid': uid = item.value.strip() elif item.name == 'poll_interval': try: poll_interval = int(item.value) assert poll_interval >= 5 except Exception: self.send_error(400, 'invalid poll interval') return # TODO: only allow from http server? uid = self._ctx.set_uid(self.client_address[0], poll_interval, uid) if not uid: self.send_error(400, 'invalid uid') return self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(uid).encode()) return elif client_request == 'set_secret': secret = None uid = None for item in form.list: if item.name == 'secret': secret = item.value.strip() elif item.name == 'uid': uid = item.value.strip() if secret and uid == self._ctx.client_uid: self._ctx.client_uid_time = time.time() self._ctx.set_secret(secret) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return elif client_request == 'add_node': host = '' port = None uid = None for item in form.list: if item.name == 'host': host = item.value elif item.name == 'port': try: port = int(item.value) except Exception: port = None elif item.name == 'uid': uid = item.value.strip() if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return if host and port: ip_addr = dispy._node_ipaddr(host) if ip_addr: info = {'ip_addr': ip_addr, 'port': port} Task(self._ctx.add_node, info) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return elif client_request == 'service_time': hosts = [] svc_time = None control = None uid = None for item in form.list: if item.name == 'hosts': hosts = [str(host) for host in json.loads(item.value)] elif item.name == 'control': control = item.value elif item.name == 'time': svc_time = item.value.strip() elif item.name == 'uid': uid = item.value.strip() if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() for host in hosts: Task(self._ctx.service_time, host, control, svc_time) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) return elif client_request == 'set_cpus': hosts = [] cpus = None uid = None for item in form.list: if item.name == 'hosts': hosts = [str(host) for host in json.loads(item.value)] if not hosts: self.send_error(400, 'invalid nodes') return elif item.name == 'cpus': cpus = item.value if cpus is not None: try: cpus = int(item.value) except Exception: self.send_error(400, 'invalid CPUs') return elif item.name == 'uid': uid = item.value.strip() if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return for host in hosts: Task(self._ctx.set_cpus, host, cpus) self._ctx.client_uid_time = time.time() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) return elif client_request == 'serve_clients': host = '' serve = None uid = None for item in form.list: if item.name == 'host': host = item.value elif item.name == 'serve': serve = item.value try: serve = int(serve) except Exception: pass elif item.name == 'uid': uid = item.value.strip() if (uid == self._ctx.client_uid and isinstance(serve, int) and Task(self._ctx.serve_clients, host, serve).value() == 0): self._ctx.client_uid_time = time.time() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return return elif client_request == 'poll_interval': uid = None interval = None for item in form.list: if item.name == 'interval': try: interval = int(item.value) except Exception: if interval is not None: logger.warning( '%s: invalid poll interval "%s" ignored', self._ctx.client_uid, item.value) self.send_error(400) return elif item.name == 'uid': uid = item.value.strip() if (uid == self._ctx.client_uid and self._ctx.set_poll_interval(interval) == 0): self._ctx.client_uid_time = time.time() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return logger.debug('Bad POST request from %s: %s', self.client_address[0], client_request) self.send_error(400) return
def do_GET(self): if self.path == '/cluster_updates': self._ctx._cluster_lock.acquire() clusters = [{ 'name': name, 'jobs': { 'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done }, 'nodes': self.__class__.json_encode_nodes(cluster.updates) } for name, cluster in dict_iter(self._ctx._clusters, 'items')] for cluster in dict_iter(self._ctx._clusters, 'values'): cluster.updates.clear() self._ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(clusters).encode()) return elif self.path == '/cluster_status': self._ctx._cluster_lock.acquire() clusters = [{ 'name': name, 'jobs': { 'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done }, 'nodes': self.__class__.json_encode_nodes(cluster.status) } for name, cluster in dict_iter(self._ctx._clusters, 'items')] self._ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(clusters).encode()) return elif self.path == '/nodes': self._ctx._cluster_lock.acquire() nodes = [{ 'name': name, 'nodes': self.__class__.json_encode_nodes(cluster.status) } for name, cluster in dict_iter(self._ctx._clusters, 'items')] self._ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return else: parsed_path = urlparse(self.path) path = parsed_path.path.lstrip('/') if path == '' or path == 'index.html': path = 'monitor.html' path = os.path.join(self.DocumentRoot, path) try: with open(path) as fd: data = fd.read() if path.endswith('.html'): if path.endswith('monitor.html') or path.endswith( 'node.html'): data = data % { 'TIMEOUT': str(self._ctx._poll_sec), 'SHOW_JOB_ARGS': 'true' if self._ctx._show_args else 'false' } content_type = 'text/html' elif path.endswith('.js'): content_type = 'text/javascript' elif path.endswith('.css'): content_type = 'text/css' elif path.endswith('.ico'): content_type = 'image/x-icon' self.send_response(200) self.send_header('Content-Type', content_type) if content_type == 'text/css' or content_type == 'text/javascript': self.send_header('Cache-Control', 'private, max-age=86400') self.end_headers() self.wfile.write(data.encode()) return except Exception: logger.warning('HTTP client %s: Could not read/send "%s"', self.client_address[0], path) logger.debug(traceback.format_exc()) self.send_error(404) return logger.debug('Bad GET request from %s: %s', self.client_address[0], self.path) self.send_error(400) return
def do_POST(self): try: form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}) client_request = self.path[1:] except Exception: logger.debug('Ignoring invalid POST request from %s', self.client_address[0]) self.send_error(400) return if client_request == 'node_info': ip_addr = None for item in form.list: if item.name == 'host': # if it looks like IP address, skip resolving if re.match(DispyHTTPServer._ClusterInfo.ip_re, item.value): ip_addr = item.value else: ip_addr = dispy._node_ipaddr(item.value) break self._ctx._cluster_lock.acquire() cluster_infos = [ (name, cluster_info) for name, cluster_info in self._ctx._clusters.items() ] self._ctx._cluster_lock.release() cluster_jobs = {} node = None show_args = self._ctx._show_args for name, cluster_info in cluster_infos: cluster_node = cluster_info.status.get(ip_addr, None) if not cluster_node: cluster_jobs[name] = [] continue if node: node.jobs_done += cluster_node.jobs_done node.cpu_time += cluster_node.cpu_time node.update_time = max(node.update_time, cluster_node.update_time) node.tx += cluster_node.tx node.rx += cluster_node.rx else: node = copy.copy(cluster_node) # jobs = cluster_info.cluster.node_jobs(ip_addr) jobs = [ job for job in dict_iter(cluster_info.jobs, 'values') if job.ip_addr == ip_addr ] # args and kwargs are sent as strings in Python, # so an object's __str__ or __repr__ is used if provided; # TODO: check job is in _ctx's jobs? jobs = [{ 'uid': job._uid, 'job_id': str(job.id), 'args': ', '.join(str(arg) for arg in job._args) if show_args else '', 'kwargs': ', '.join('%s=%s' % (key, val) for key, val in job._kwargs.items()) if show_args else '', 'start_time_ms': int(1000 * job.start_time), 'cluster': name } for job in jobs] cluster_jobs[name] = jobs self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() if node: if node.avail_info: node.avail_info = node.avail_info.__dict__ self.wfile.write( json.dumps({ 'node': node.__dict__, 'cluster_jobs': cluster_jobs }).encode()) return elif client_request == 'cancel_jobs': uids = [] for item in form.list: if item.name == 'uid': try: uids.append(int(item.value)) except ValueError: logger.debug('Cancel job uid "%s" is invalid', item.value) self._ctx._cluster_lock.acquire() cluster_jobs = [ (cluster_info.cluster, cluster_info.jobs.get(uid, None)) for cluster_info in self._ctx._clusters.values() for uid in uids ] self._ctx._cluster_lock.release() cancelled = [] for cluster, job in cluster_jobs: if not job: continue if cluster.cancel(job) == 0: cancelled.append(job._uid) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(cancelled).encode()) return elif client_request == 'add_node': node = {'host': '', 'port': None, 'cpus': 0, 'cluster': None} node_id = None cluster = None for item in form.list: if item.name == 'host': node['host'] = item.value elif item.name == 'cluster': node['cluster'] = item.value elif item.name == 'port': node['port'] = item.value elif item.name == 'cpus': try: node['cpus'] = int(item.value) except Exception: pass elif item.name == 'id': node_id = item.value if node['host']: self._ctx._cluster_lock.acquire() clusters = [ cluster_info.cluster for name, cluster_info in self._ctx._clusters.items() if name == node['cluster'] or not node['cluster'] ] self._ctx._cluster_lock.release() for cluster in clusters: cluster.allocate_node(node) self.send_response(200) self.send_header('Content-Type', 'text/html') self.end_headers() node['id'] = node_id self.wfile.write(json.dumps(node).encode()) return elif (client_request == 'close_node' or client_request == 'allocate_node' or client_request == 'deallocate_node'): nodes = [] cluster_infos = [] resp = -1 for item in form.list: if item.name == 'cluster': self._ctx._cluster_lock.acquire() if item.value == '*': cluster_infos = list(self._ctx._clusters.values()) else: cluster_infos = [ self._ctx._clusters.get(item.value, None) ] if not cluster_infos[0]: cluster_infos = [] self._ctx._cluster_lock.release() elif item.name == 'nodes': nodes = json.loads(item.value) nodes = [str(node) for node in nodes] if cluster_infos and nodes: resp = 0 for cluster_info in cluster_infos: fn = getattr(cluster_info.cluster, client_request) if fn: for node in nodes: resp |= fn(node) else: resp = -1 self.send_response(200) self.send_header( 'Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(resp).encode()) return elif client_request == 'update': for item in form.list: if item.name == 'timeout': try: timeout = int(item.value) if timeout < 1: timeout = 0 self._ctx._poll_sec = timeout except Exception: logger.warning( 'HTTP client %s: invalid timeout "%s" ignored', self.client_address[0], item.value) elif item.name == 'show_job_args': if item.value == 'true': self._ctx._show_args = True else: self._ctx._show_args = False return elif client_request == 'set_cpus': node_cpus = {} for item in form.list: self._ctx._cluster_lock.acquire() for cluster_info in self._ctx._clusters.values(): node = cluster_info.status.get(item.name, None) if node: node_cpus[ item. name] = cluster_info.cluster.set_node_cpus( item.name, item.value) if node_cpus[item.name] >= 0: break self._ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(node_cpus).encode()) return logger.debug('Bad POST request from %s: %s', self.client_address[0], client_request) self.send_error(400) return
def do_GET(self): if self.path == '/cluster_updates': self._dispy_ctx._cluster_lock.acquire() clusters = [ {'name': name, 'jobs': {'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done}, 'nodes': self.__class__.json_encode_nodes(cluster.updates) } for name, cluster in dict_iter(self._dispy_ctx._clusters, 'items') ] for cluster in dict_iter(self._dispy_ctx._clusters, 'values'): cluster.updates.clear() self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(clusters).encode()) return elif self.path == '/cluster_status': self._dispy_ctx._cluster_lock.acquire() clusters = [ {'name': name, 'jobs': {'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done}, 'nodes': self.__class__.json_encode_nodes(cluster.status) } for name, cluster in dict_iter(self._dispy_ctx._clusters, 'items') ] self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(clusters).encode()) return elif self.path == '/nodes': self._dispy_ctx._cluster_lock.acquire() nodes = [ {'name': name, 'nodes': self.__class__.json_encode_nodes(cluster.status) } for name, cluster in dict_iter(self._dispy_ctx._clusters, 'items') ] self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return else: parsed_path = urlparse(self.path) path = parsed_path.path.lstrip('/') if path == '' or path == 'index.html': path = 'monitor.html' path = os.path.join(self.DocumentRoot, path) try: with open(path) as fd: data = fd.read() if path.endswith('.html'): if path.endswith('monitor.html') or path.endswith('node.html'): data = data % {'TIMEOUT': str(self._dispy_ctx._poll_sec)} content_type = 'text/html' elif path.endswith('.js'): content_type = 'text/javascript' elif path.endswith('.css'): content_type = 'text/css' elif path.endswith('.ico'): content_type = 'image/x-icon' self.send_response(200) self.send_header('Content-Type', content_type) if content_type == 'text/css' or content_type == 'text/javascript': self.send_header('Cache-Control', 'private, max-age=86400') self.end_headers() self.wfile.write(data.encode()) return except: logger.warning('HTTP client %s: Could not read/send "%s"', self.client_address[0], path) logger.debug(traceback.format_exc()) self.send_error(404) return logger.debug('Bad GET request from %s: %s' % (self.client_address[0], self.path)) self.send_error(400) return
def do_POST(self): form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}) if self.path == '/node_jobs': ip_addr = None for item in form.list: if item.name == 'host': # if it looks like IP address, skip resolving if re.match('^\d+[\.\d]+$', item.value): ip_addr = item.value else: try: ip_addr = socket.gethostbyname(item.value) except: ip_addr = item.value break self._dispy_ctx._cluster_lock.acquire() cluster_infos = [(name, cluster_info) for name, cluster_info in self._dispy_ctx._clusters.items()] self._dispy_ctx._cluster_lock.release() jobs = [] node = None for name, cluster_info in cluster_infos: cluster_node = cluster_info.status.get(ip_addr, None) if not cluster_node: continue if node: node.jobs_done += cluster_node.jobs_done node.cpu_time += cluster_node.cpu_time node.update_time = max(node.update_time, cluster_node.update_time) else: node = copy.copy(cluster_node) cluster_jobs = cluster_info.cluster.node_jobs(ip_addr) # args and kwargs are sent as strings in Python, # so an object's __str__ or __repr__ is used if provided; # TODO: check job is in _dispy_ctx's jobs? jobs.extend([{'uid': id(job), 'job_id': str(job.id), 'args': ', '.join(str(arg) for arg in job.args), 'kwargs': ', '.join('%s=%s' % (key, val) for key, val in job.kwargs.items()), 'sched_time_ms': int(1000 * job.start_time), 'cluster': name} for job in cluster_jobs]) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() if node and node.avail_info: node.avail_info = node.avail_info.__dict__ self.wfile.write(json.dumps({'node': node.__dict__, 'jobs': jobs}).encode()) return elif self.path == '/cancel_jobs': uids = [] for item in form.list: if item.name == 'uid': try: uids.append(int(item.value)) except ValueError: logger.debug('Cancel job uid "%s" is invalid' % item.value) self._dispy_ctx._cluster_lock.acquire() cluster_jobs = [(cluster_info.cluster, cluster_info.jobs.get(uid, None)) for cluster_info in self._dispy_ctx._clusters.values() for uid in uids] self._dispy_ctx._cluster_lock.release() cancelled = [] for cluster, job in cluster_jobs: if not job: continue if cluster.cancel(job) == 0: cancelled.append(id(job)) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(cancelled).encode()) return elif self.path == '/add_node': node = {'host': '', 'port': None, 'cpus': 0, 'cluster': None} node_id = None cluster = None for item in form.list: if item.name == 'host': node['host'] = item.value elif item.name == 'cluster': node['cluster'] = item.value elif item.name == 'port': node['port'] = item.value elif item.name == 'cpus': try: node['cpus'] = int(item.value) except: pass elif item.name == 'id': node_id = item.value if node['host']: self._dispy_ctx._cluster_lock.acquire() clusters = [cluster_info.cluster for name, cluster_info in self._dispy_ctx._clusters.items() if name == node['cluster'] or not node['cluster']] self._dispy_ctx._cluster_lock.release() for cluster in clusters: cluster.allocate_node(node) self.send_response(200) self.send_header('Content-Type', 'text/html') self.end_headers() node['id'] = node_id self.wfile.write(json.dumps(node).encode()) return elif self.path == '/set_poll_sec': for item in form.list: if item.name != 'timeout': continue try: timeout = int(item.value) if timeout < 1: timeout = 0 except: logger.warning('HTTP client %s: invalid timeout "%s" ignored', self.client_address[0], item.value) timeout = 0 self._dispy_ctx._poll_sec = timeout self.send_response(200) self.send_header('Content-Type', 'text/html') self.end_headers() return elif self.path == '/set_cpus': node_cpus = {} for item in form.list: self._dispy_ctx._cluster_lock.acquire() for cluster_info in self._dispy_ctx._clusters.values(): node = cluster_info.status.get(item.name, None) if node: node_cpus[item.name] = cluster_info.cluster.set_node_cpus( item.name, item.value) if node_cpus[item.name] >= 0: break self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(node_cpus).encode()) return logger.debug('Bad POST request from %s: %s' % (self.client_address[0], self.path)) self.send_error(400) return
def udp_server(self, addrinfo, task=None): task.set_daemon() udp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM)) udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): try: udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) except Exception: pass udp_sock.bind((addrinfo.bind_addr, self.info_port)) if addrinfo.family == socket.AF_INET: if self.ipv4_udp_multicast: mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(addrinfo.ip) udp_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) else: # addrinfo.family == socket.AF_INET6: mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast) mreq += struct.pack('@I', addrinfo.ifn) udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq) try: udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) except Exception: pass while 1: msg, addr = yield udp_sock.recvfrom(1000) if msg.startswith('PING:'): try: info = deserialize(msg[len('PING:'):]) if info['version'] != _dispy_version: logger.warning('Ignoring %s due to version mismatch', addr[0]) continue assert info['port'] > 0 assert info['ip_addr'] except Exception: logger.debug('Ignoring node %s', addr[0]) continue node = self.nodes.get(info['ip_addr'], None) if node: if node._priv.sign == info['sign']: Task(self.update_node_info, node) else: node._priv.sign = info['sign'] node._priv.auth = None Task(self.get_node_info, node) else: info['family'] = addrinfo.family Task(self.add_node, info) elif msg.startswith('TERMINATED:'): try: info = deserialize(msg[len('TERMINATED:'):]) assert info['ip_addr'] except Exception: logger.debug('Ignoring node %s', addr[0]) continue node = self.nodes.get(info['ip_addr'], None) if node and node._priv.sign == info['sign']: with self.lock: self.nodes.pop(info['ip_addr'], None)
def do_POST(self): try: form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}) client_request = self.path[1:] except Exception: logger.debug('Ignoring invalid POST request from %s', self.client_address[0]) self.send_error(400) return if client_request == 'update': uid = None for item in form.list: if item.name == 'uid': uid = item.value.strip() break if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() self._ctx.lock.acquire() nodes = self.__class__.json_encode_nodes(self._ctx.updates) self._ctx.updates.clear() self._ctx.lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return elif client_request == 'node_info': ip_addr = None uid = None for item in form.list: if item.name == 'host': # if it looks like IP address, skip resolving if re.match(DispyAdminServer._NodeInfo.ip_re, item.value): ip_addr = item.value else: ip_addr = dispy._node_ipaddr(item.value) elif item.name == 'uid': uid = item.value.strip() if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() node = self._ctx.nodes.get(ip_addr, None) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() if node: node = dict(node.__dict__) node.pop('_priv', None) if node['avail_info']: node['avail_info'] = node['avail_info'].__dict__ else: node = {} self.wfile.write(json.dumps(node).encode()) return elif client_request == 'status': uid = None for item in form.list: if item.name == 'uid': uid = item.value.strip() break if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() self._ctx.lock.acquire() nodes = self.__class__.json_encode_nodes(self._ctx.nodes) self._ctx.lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return elif client_request == 'get_uid': uid = None for item in form.list: if item.name == 'uid': uid = item.value.strip() break now = time.time() # TODO: only allow from http server? if (self._ctx.client_uid and uid != self._ctx.client_uid and ((now - self._ctx.client_uid_time) < 3600)): self.send_error(400, 'invalid uid') return if not uid: uid = hashlib.sha1(os.urandom(20)).hexdigest() self._ctx.client_uid = uid self._ctx.client_uid_time = time.time() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(uid).encode()) return elif client_request == 'set_secret': secret = None uid = None for item in form.list: if item.name == 'secret': secret = item.value.strip() elif item.name == 'uid': uid = item.value.strip() if secret and uid == self._ctx.client_uid: self._ctx.client_uid_time = time.time() self._ctx.set_secret(secret) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return elif client_request == 'add_node': host = '' port = None uid = None for item in form.list: if item.name == 'host': host = item.value elif item.name == 'port': try: port = int(item.value) except Exception: port = None elif item.name == 'uid': uid = item.value.strip() if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return if host and port: ip_addr = dispy._node_ipaddr(host) if ip_addr: info = {'ip_addr': ip_addr, 'port': port} Task(self._ctx.add_node, info) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return elif client_request == 'service_time': hosts = [] svc_time = None control = None uid = None for item in form.list: if item.name == 'hosts': hosts = [str(host) for host in json.loads(item.value)] elif item.name == 'control': control = item.value elif item.name == 'time': svc_time = item.value elif item.name == 'uid': uid = item.value.strip() if uid != self._ctx.client_uid: self.send_error(400, 'invalid uid') return self._ctx.client_uid_time = time.time() for host in hosts: Task(self._ctx.service_time, host, control, svc_time) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) return elif client_request == 'set_cpus': host = None cpus = None uid = None for item in form.list: if item.name == 'host': host = item.value.strip() elif item.name == 'cpus': try: cpus = int(item.value.strip()) except Exception: pass elif item.name == 'uid': uid = item.value.strip() if (host and cpus and uid == self._ctx.client_uid and Task(self._ctx.set_cpus, host, cpus).value() == 0): self._ctx.client_uid_time = time.time() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return elif client_request == 'serve_clients': host = '' serve = None uid = None for item in form.list: if item.name == 'host': host = item.value elif item.name == 'serve': serve = item.value try: serve = int(serve) except Exception: pass elif item.name == 'uid': uid = item.value.strip() if (uid == self._ctx.client_uid and isinstance(serve, int) and Task(self._ctx.serve_clients, host, serve).value() == 0): self._ctx.client_uid_time = time.time() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return return elif client_request == 'poll_interval': uid = None interval = None for item in form.list: if item.name == 'interval': try: interval = int(item.value) except Exception: if interval is not None: logger.warning('%s: invalid poll interval "%s" ignored', self._ctx.client_uid, item.value) self.send_error(400) return elif item.name == 'uid': uid = item.value.strip() if (uid == self._ctx.client_uid and self._ctx.set_poll_interval(interval) == 0): self._ctx.client_uid_time = time.time() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(0).encode()) else: self.send_error(400) return logger.debug('Bad POST request from %s: %s', self.client_address[0], client_request) self.send_error(400) return
def do_GET(self): if self.path == '/cluster_updates': self._dispy_ctx._cluster_lock.acquire() clusters = [{ 'name': name, 'jobs': { 'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done }, 'nodes': self.__class__.json_encode_nodes(cluster.updates) } for name, cluster in dict_iter( self._dispy_ctx._clusters, 'items')] for cluster in dict_iter(self._dispy_ctx._clusters, 'values'): cluster.updates.clear() self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(clusters).encode()) return elif self.path == '/cluster_status': self._dispy_ctx._cluster_lock.acquire() clusters = [{ 'name': name, 'jobs': { 'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done }, 'nodes': self.__class__.json_encode_nodes(cluster.status) } for name, cluster in dict_iter( self._dispy_ctx._clusters, 'items')] self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(clusters).encode()) return elif self.path == '/all_nodes_list': now = time.time() self._dispy_ctx._cluster_lock.acquire() if self._dispy_ctx.all_available_nodes: nodes = { key: { "update_time": node.last_pulse, "avail_cpus": node.avail_cpus, "name": node.name, "cpus": node.cpus, "avail_info": node.avail_info.__dict__, "ip_addr": node.ip_addr, "busy": node.busy } for key, node in self._dispy_ctx.all_available_nodes.items() if node.last_pulse and (now - node.last_pulse) <= 60 * 10 } else: nodes = {} self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return elif self.path == '/nodes': self._dispy_ctx._cluster_lock.acquire() nodes = [{ 'name': name, 'nodes': self.__class__.json_encode_nodes(cluster.status) } for name, cluster in dict_iter( self._dispy_ctx._clusters, 'items')] self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(nodes).encode()) return else: parsed_path = urlparse(self.path) path = parsed_path.path.lstrip('/') if path == '' or path == 'index.html': path = 'monitor.html' path = os.path.join(self.DocumentRoot, path) try: with open(path) as fd: data = fd.read() if path.endswith('.html'): if path.endswith('monitor.html') or path.endswith( 'node.html') or path.endswith( 'available_nodes.html'): data = data % { 'TIMEOUT': str(self._dispy_ctx._poll_sec) } content_type = 'text/html' elif path.endswith('.js'): content_type = 'text/javascript' elif path.endswith('.css'): content_type = 'text/css' elif path.endswith('.ico'): content_type = 'image/x-icon' self.send_response(200) self.send_header('Content-Type', content_type) if content_type == 'text/css' or content_type == 'text/javascript': self.send_header('Cache-Control', 'private, max-age=86400') self.end_headers() self.wfile.write(data.encode()) return except: logger.warning('HTTP client %s: Could not read/send "%s"', self.client_address[0], path) logger.debug(traceback.format_exc()) self.send_error(404) return logger.debug('Bad GET request from %s: %s', self.client_address[0], self.path) self.send_error(400) return
def do_POST(self): form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}) if self.path == '/node_jobs': ip_addr = None for item in form.list: if item.name == 'host': # if it looks like IP address, skip resolving if re.match('^\d+[\.\d]+$', item.value): ip_addr = item.value else: try: ip_addr = socket.gethostbyname(item.value) except: ip_addr = item.value break self._dispy_ctx._cluster_lock.acquire() cluster_infos = [(name, cluster_info) for name, cluster_info in self._dispy_ctx._clusters.items()] self._dispy_ctx._cluster_lock.release() jobs = [] node = None for name, cluster_info in cluster_infos: cluster_node = cluster_info.status.get(ip_addr, None) if not cluster_node: continue if node: node.jobs_done += cluster_node.jobs_done node.cpu_time += cluster_node.cpu_time node.update_time = max(node.update_time, cluster_node.update_time) else: node = copy.copy(cluster_node) cluster_jobs = cluster_info.cluster.node_jobs(ip_addr) # args and kwargs are sent as strings in Python, # so an object's __str__ or __repr__ is used if provided; # TODO: check job is in _dispy_ctx's jobs? jobs.extend([{ 'uid': id(job), 'job_id': str(job.id), 'args': ', '.join(str(arg) for arg in job.args), 'kwargs': ', '.join('%s=%s' % (key, val) for key, val in job.kwargs.items()), 'sched_time_ms': int(1000 * job.start_time), 'cluster': name } for job in cluster_jobs]) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() if node and node.avail_info: node.avail_info = node.avail_info.__dict__ self.wfile.write( json.dumps({ 'node': node.__dict__, 'jobs': jobs }).encode()) return elif self.path == '/cancel_jobs': uids = [] for item in form.list: if item.name == 'uid': try: uids.append(int(item.value)) except ValueError: logger.debug('Cancel job uid "%s" is invalid', item.value) self._dispy_ctx._cluster_lock.acquire() cluster_jobs = [ (cluster_info.cluster, cluster_info.jobs.get(uid, None)) for cluster_info in self._dispy_ctx._clusters.values() for uid in uids ] self._dispy_ctx._cluster_lock.release() cancelled = [] for cluster, job in cluster_jobs: if not job: continue if cluster.cancel(job) == 0: cancelled.append(id(job)) self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(cancelled).encode()) return elif self.path == '/terminate_cluster': names_of_clusters = [i.value for i in form.list] del_jobs = [ (cluster_info, job_id) for cluster_info in self._dispy_ctx._clusters.values() if cluster_info.cluster.name in names_of_clusters for job_id in cluster_info.jobs.keys() ] ret_values = [ cluster_info.cluster.cancel(cluster_info.jobs[job_id]) for cluster_info, job_id in del_jobs ] delete_clusters = [ cluster_info.cluster for cluster_info in self._dispy_ctx._clusters.values() if cluster_info.cluster.name in names_of_clusters ] [self._dispy_ctx.del_cluster(cl) for cl in delete_clusters] if sum(ret_values) == 0: self.send_response(200) self.send_header('Content-Type', 'text/plain; charset=utf-8') self.end_headers() self.wfile.write("OK".encode()) return elif self.path == '/add_node': node = {'host': '', 'port': None, 'cpus': 0, 'cluster': None} node_id = None cluster = None for item in form.list: if item.name == 'host': node['host'] = item.value elif item.name == 'cluster': node['cluster'] = item.value elif item.name == 'port': node['port'] = item.value elif item.name == 'cpus': try: node['cpus'] = int(item.value) except: pass elif item.name == 'id': node_id = item.value if node['host']: self._dispy_ctx._cluster_lock.acquire() clusters = [ cluster_info.cluster for name, cluster_info in self._dispy_ctx._clusters.items() if name == node['cluster'] or not node['cluster'] ] self._dispy_ctx._cluster_lock.release() for cluster in clusters: cluster.allocate_node(node) self.send_response(200) self.send_header('Content-Type', 'text/html') self.end_headers() node['id'] = node_id self.wfile.write(json.dumps(node).encode()) return elif self.path == '/set_poll_sec': for item in form.list: if item.name != 'timeout': continue try: timeout = int(item.value) if timeout < 1: timeout = 0 except: logger.warning( 'HTTP client %s: invalid timeout "%s" ignored', self.client_address[0], item.value) timeout = 0 self._dispy_ctx._poll_sec = timeout self.send_response(200) self.send_header('Content-Type', 'text/html') self.end_headers() return elif self.path == '/set_cpus': node_cpus = {} for item in form.list: self._dispy_ctx._cluster_lock.acquire() for cluster_info in self._dispy_ctx._clusters.values(): node = cluster_info.status.get(item.name, None) if node: node_cpus[ item. name] = cluster_info.cluster.set_node_cpus( item.name, item.value) if node_cpus[item.name] >= 0: break self._dispy_ctx._cluster_lock.release() self.send_response(200) self.send_header('Content-Type', 'application/json; charset=utf-8') self.end_headers() self.wfile.write(json.dumps(node_cpus).encode()) return logger.debug('Bad POST request from %s: %s', self.client_address[0], self.path) self.send_error(400) return