def __init__(self, cluster, host='', port=8181, poll_sec=10, DocumentRoot=None,
keyfile=None, certfile=None):
self._cluster_lock = threading.Lock()
self._clusters = {}
if cluster:
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
if cluster.status_callback is None:
cluster.status_callback = functools.partial(self.cluster_status, cluster_info)
if not DocumentRoot:
DocumentRoot = os.path.join(os.path.dirname(__file__), 'data')
if poll_sec < 1:
logger.warning('invalid poll_sec value %s; it must be at least 1' % poll_sec)
poll_sec = 1
self._poll_sec = poll_sec
self._http_handler = None
self._server = HTTPServer((host, port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s' %
('s' if certfile else '', str(self._server.socket.getsockname())))
def set_uid(self, client_host, poll_interval, uid=None):
now = time.time()
try:
poll_interval = int(poll_interval)
assert poll_interval >= 5
except Exception:
return None
if ((uid == self.client_uid) or (not self.client_uid)
or (self.client_host == client_host) or
((now - self.client_uid_time) > min(5 * self.poll_interval, 600))):
if not uid:
uid = hashlib.sha1(os.urandom(20)).hexdigest()
self.client_uid = uid
self.client_uid_time = now
self.client_host = client_host
if self.poll_interval != poll_interval:
self.poll_interval = poll_interval
self.timer.resume(update=poll_interval)
return uid
else:
logger.warning(
'Ignoring client at %s; currently controlled by client at %s',
client_host, self.client_host)
return None
def __init__(self, DocumentRoot, secret='', http_host='localhost', http_port=8181,
info_port=51347, node_port=51348, poll_interval=60, ping_interval=600,
ip_addrs=[], ipv4_udp_multicast=False, certfile=None, keyfile=None):
self.lock = threading.Lock()
self.client_uid = None
self.client_uid_time = 0
self.nodes = {}
self.updates = {}
if poll_interval < 1:
logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval)
poll_interval = 1
self.info_port = info_port
self.poll_interval = poll_interval
self.ping_interval = ping_interval
self.secret = secret
self.node_port = node_port
self.keyfile = keyfile
self.certfile = certfile
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
self.addrinfos = {}
if not ip_addrs:
ip_addrs = [None]
for i in range(len(ip_addrs)):
ip_addr = ip_addrs[i]
addrinfo = dispy.host_addrinfo(host=ip_addr, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid ip_addr %s', ip_addr)
continue
self.addrinfos[addrinfo.ip] = addrinfo
if not self.addrinfos:
raise Exception('No valid IP address found')
self.http_port = http_port
self.sign = hashlib.sha1(os.urandom(20))
for ip_addr in self.addrinfos:
self.sign.update(ip_addr.encode())
self.sign = self.sign.hexdigest()
self.auth = dispy.auth_code(self.secret, self.sign)
self.tcp_tasks = []
self.udp_tasks = []
udp_addrinfos = {}
for addrinfo in self.addrinfos.values():
self.tcp_tasks.append(Task(self.tcp_server, addrinfo))
udp_addrinfos[addrinfo.bind_addr] = addrinfo
for bind_addr, addrinfo in udp_addrinfos.items():
self.udp_tasks.append(Task(self.udp_server, addrinfo))
self._server = HTTPServer((http_host, http_port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self.timer = Task(self.timer_proc)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s', 's' if certfile else '',
':'.join(map(str, self._server.socket.getsockname())))
def tcp_server(self, addrinfo, task=None):
task.set_daemon()
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((addrinfo.ip, self.info_port))
except Exception:
logger.warning('Could not bind TCP server to %s:%s', addrinfo.ip, self.info_port)
raise StopIteration
logger.debug('dispyadmin TCP server at %s:%s', addrinfo.ip, self.info_port)
sock.listen(16)
while 1:
try:
conn, addr = yield sock.accept()
except ssl.SSLError as err:
logger.debug('SSL connection failed: %s', str(err))
continue
except GeneratorExit:
break
except Exception:
logger.debug(traceback.format_exc())
continue
Task(self.tcp_req, conn, addr)
sock.close()
def do_GET(self):
path = urlparse(self.path).path.lstrip('/')
if path == '' or path == 'index.html':
path = 'admin.html'
path = os.path.join(self.DocumentRoot, path)
try:
with open(path) as fd:
data = fd.read()
if path.endswith('.html'):
if path.endswith('admin.html') or path.endswith(
'admin_node.html'):
data = data % {
'POLL_INTERVAL': str(self._ctx.poll_interval),
'NODE_PORT': str(self._ctx.node_port)
}
content_type = 'text/html'
elif path.endswith('.js'):
content_type = 'text/javascript'
elif path.endswith('.css'):
content_type = 'text/css'
elif path.endswith('.ico'):
content_type = 'image/x-icon'
self.send_response(200)
self.send_header('Content-Type', content_type)
if content_type == 'text/css' or content_type == 'text/javascript':
self.send_header('Cache-Control', 'private, max-age=86400')
self.end_headers()
self.wfile.write(data.encode())
return
except Exception:
logger.warning('HTTP client %s: Could not read/send "%s"',
self.client_address[0], path)
logger.debug(traceback.format_exc())
self.send_error(404)
return
def do_GET(self):
parsed_path = urlparse(self.path)
path = parsed_path.path.lstrip('/')
if path == '' or path == 'index.html':
path = 'admin.html'
path = os.path.join(self.DocumentRoot, path)
try:
with open(path) as fd:
data = fd.read()
if path.endswith('.html'):
if path.endswith('admin.html') or path.endswith('admin_node.html'):
data = data % {'POLL_INTERVAL': str(self._ctx.poll_interval),
'NODE_PORT': str(self._ctx.node_port)}
content_type = 'text/html'
elif path.endswith('.js'):
content_type = 'text/javascript'
elif path.endswith('.css'):
content_type = 'text/css'
elif path.endswith('.ico'):
content_type = 'image/x-icon'
self.send_response(200)
self.send_header('Content-Type', content_type)
if content_type == 'text/css' or content_type == 'text/javascript':
# self.send_header('Cache-Control', 'private, max-age=86400')
self.send_header('Cache-Control', 'private, max-age=30')
self.end_headers()
self.wfile.write(data.encode())
return
except Exception:
logger.warning('HTTP client %s: Could not read/send "%s"',
self.client_address[0], path)
logger.debug(traceback.format_exc())
self.send_error(404)
return
def tcp_server(self, addrinfo, task=None):
task.set_daemon()
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((addrinfo.ip, self.info_port))
except Exception:
logger.warning('Could not bind TCP server to %s:%s', addrinfo.ip,
self.info_port)
raise StopIteration
logger.info('dispyadmin TCP server at %s:%s', addrinfo.ip,
self.info_port)
sock.listen(16)
while 1:
try:
conn, addr = yield sock.accept()
except ssl.SSLError as err:
logger.debug('SSL connection failed: %s', str(err))
continue
except GeneratorExit:
break
except Exception:
logger.debug(traceback.format_exc())
continue
Task(self.tcp_req, conn, addr)
sock.close()
def __init__(self, cluster, host='', port=8181, poll_sec=10, DocumentRoot=None,
keyfile=None, certfile=None):
self._cluster_lock = threading.Lock()
self._clusters = {}
if cluster:
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
if cluster.status_callback is None:
cluster.status_callback = functools.partial(self.cluster_status, cluster_info)
if not DocumentRoot:
DocumentRoot = os.path.join(os.path.dirname(__file__), 'data')
if poll_sec < 1:
logger.warning('invalid poll_sec value %s; it must be at least 1' % poll_sec)
poll_sec = 1
self._poll_sec = poll_sec
self._http_handler = None
self._server = HTTPServer((host, port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s' %
('s' if certfile else '', str(self._server.socket.getsockname())))
def __init__(self, DocumentRoot, secret='', http_host='localhost',
poll_interval=60, ping_interval=600, hosts=[], ipv4_udp_multicast=False,
certfile=None, keyfile=None):
http_port = dispy.config.HTTPServerPort
self.node_port = eval(dispy.config.NodePort)
self.info_port = eval(dispy.config.ClientPort)
self.lock = threading.Lock()
self.client_uid = None
self.client_uid_time = 0
self.nodes = {}
self.updates = {}
if poll_interval < 1:
logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval)
poll_interval = 1
self.poll_interval = poll_interval
self.ping_interval = ping_interval
self.secret = secret
self.keyfile = keyfile
self.certfile = certfile
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
self.addrinfos = []
if not hosts:
hosts = [None]
for host in hosts:
addrinfo = dispy.host_addrinfo(host=host, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid host %s', host)
continue
self.addrinfos.append(addrinfo)
if not self.addrinfos:
raise Exception('No valid host name / IP address found')
self.sign = hashlib.sha1(os.urandom(20))
for addrinfo in self.addrinfos:
self.sign.update(addrinfo.ip.encode())
self.sign = self.sign.hexdigest()
self.auth = dispy.auth_code(self.secret, self.sign)
self.tcp_tasks = []
self.udp_tasks = []
udp_addrinfos = {}
for addrinfo in self.addrinfos:
self.tcp_tasks.append(Task(self.tcp_server, addrinfo))
udp_addrinfos[addrinfo.bind_addr] = addrinfo
for bind_addr, addrinfo in udp_addrinfos.items():
self.udp_tasks.append(Task(self.udp_server, addrinfo))
self._server = HTTPServer((http_host, http_port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self.timer = Task(self.timer_proc)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
self.client_host = self._server.socket.getsockname()[0]
logger.info('Started HTTP%s server at %s:%s', 's' if certfile else '',
self.client_host, self._server.socket.getsockname()[1])
def add_cluster(self, cluster):
"""If more than one cluster is used in a program, they can be
added to http server for monitoring.
"""
if cluster.name in self._clusters:
logger.warning('Cluster "%s" is already registered' % (cluster.name))
return
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
if cluster.status_callback is None:
cluster.status_callback = functools.partial(self.cluster_status, cluster_info)
def add_cluster(self, cluster):
"""If more than one cluster is used in a program, they can be
added to http server for monitoring.
"""
if cluster.name in self._clusters:
logger.warning('Cluster "%s" is already registered' % (cluster.name))
return
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
if cluster.status_callback is None:
cluster.status_callback = functools.partial(self.cluster_status, cluster_info)
def __init__(self,
cluster,
host='',
port=int(dispy.config.HTTPServerPort),
poll_sec=10,
DocumentRoot=None,
keyfile=None,
certfile=None,
show_job_args=True):
self._cluster_lock = threading.Lock()
self._clusters = {}
if cluster:
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
http_callback = functools.partial(self.cluster_status,
cluster_info)
if cluster.status_callback:
client_callback = cluster.status_callback
def chain_callbacks(status, node, job):
http_callback(status, node, job)
client_callback(status, node, job)
cluster.status_callback = chain_callbacks
else:
cluster.status_callback = http_callback
if not DocumentRoot:
DocumentRoot = os.path.join(os.path.dirname(__file__), 'data')
if poll_sec < 1:
logger.warning('invalid poll_sec value %s; it must be at least 1',
poll_sec)
poll_sec = 1
self._poll_sec = poll_sec
self._show_args = bool(show_job_args)
self._server = HTTPServer(
(host, int(port)),
lambda *args: self.__class__._HTTPRequestHandler(
self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket,
keyfile=keyfile,
certfile=certfile,
server_side=True)
self._httpd_thread = threading.Thread(
target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s', 's' if certfile else '',
str(self._server.socket.getsockname()))
def udp_server(self, addrinfo, task=None):
task.set_daemon()
udp_sock = AsyncSocket(
socket.socket(addrinfo.family, socket.SOCK_DGRAM))
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
pass
udp_sock.bind((addrinfo.bind_addr, self.info_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(
addrinfo.ip)
udp_sock.setsockopt(socket.IPPROTO_IP,
socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP,
mreq)
try:
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
except Exception:
pass
while 1:
msg, addr = yield udp_sock.recvfrom(1000)
if msg.startswith(b'PING:'):
try:
info = deserialize(msg[len(b'PING:'):])
if info['version'] != _dispy_version:
logger.warning('Ignoring %s due to version mismatch',
addr[0])
continue
assert info['port'] > 0
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node:
if node._priv.sign == info['sign']:
Task(self.update_node_info, node)
else:
node._priv.sign = info['sign']
node._priv.auth = None
Task(self.get_node_info, node)
else:
info['family'] = addrinfo.family
Task(self.add_node, info)
elif msg.startswith(b'TERMINATED:'):
try:
info = deserialize(msg[len(b'TERMINATED:'):])
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node and node._priv.sign == info['sign']:
with self.lock:
self.nodes.pop(info['ip_addr'], None)
def do_GET(self):
if self.path == '/cluster_updates':
self._dispy_ctx._cluster_lock.acquire()
clusters = [{
'name':
name,
'jobs': {
'submitted': cluster.jobs_submitted,
'done': cluster.jobs_done
},
'nodes':
self.__class__.json_encode_nodes(cluster.updates)
}
for name, cluster in dict_iter(
self._dispy_ctx._clusters, 'items')]
for cluster in dict_iter(self._dispy_ctx._clusters, 'values'):
cluster.updates.clear()
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(clusters).encode())
return
elif self.path == '/cluster_status':
self._dispy_ctx._cluster_lock.acquire()
clusters = [{
'name':
name,
'jobs': {
'submitted': cluster.jobs_submitted,
'done': cluster.jobs_done
},
'nodes':
self.__class__.json_encode_nodes(cluster.status)
}
for name, cluster in dict_iter(
self._dispy_ctx._clusters, 'items')]
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(clusters).encode())
return
elif self.path == '/all_nodes_list':
now = time.time()
self._dispy_ctx._cluster_lock.acquire()
if self._dispy_ctx.all_available_nodes:
nodes = {
key: {
"update_time": node.last_pulse,
"avail_cpus": node.avail_cpus,
"name": node.name,
"cpus": node.cpus,
"avail_info": node.avail_info.__dict__,
"ip_addr": node.ip_addr,
"busy": node.busy
}
for key, node in
self._dispy_ctx.all_available_nodes.items()
if node.last_pulse and (now -
node.last_pulse) <= 60 * 10
}
else:
nodes = {}
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif self.path == '/nodes':
self._dispy_ctx._cluster_lock.acquire()
nodes = [{
'name':
name,
'nodes':
self.__class__.json_encode_nodes(cluster.status)
}
for name, cluster in dict_iter(
self._dispy_ctx._clusters, 'items')]
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
else:
parsed_path = urlparse(self.path)
path = parsed_path.path.lstrip('/')
if path == '' or path == 'index.html':
path = 'monitor.html'
path = os.path.join(self.DocumentRoot, path)
try:
with open(path) as fd:
data = fd.read()
if path.endswith('.html'):
if path.endswith('monitor.html') or path.endswith(
'node.html') or path.endswith(
'available_nodes.html'):
data = data % {
'TIMEOUT': str(self._dispy_ctx._poll_sec)
}
content_type = 'text/html'
elif path.endswith('.js'):
content_type = 'text/javascript'
elif path.endswith('.css'):
content_type = 'text/css'
elif path.endswith('.ico'):
content_type = 'image/x-icon'
self.send_response(200)
self.send_header('Content-Type', content_type)
if content_type == 'text/css' or content_type == 'text/javascript':
self.send_header('Cache-Control',
'private, max-age=86400')
self.end_headers()
self.wfile.write(data.encode())
return
except:
logger.warning('HTTP client %s: Could not read/send "%s"',
self.client_address[0], path)
logger.debug(traceback.format_exc())
self.send_error(404)
return
logger.debug('Bad GET request from %s: %s', self.client_address[0],
self.path)
self.send_error(400)
return
def do_POST(self):
try:
form = cgi.FieldStorage(fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
client_request = self.path[1:]
except Exception:
logger.debug('Ignoring invalid POST request from %s',
self.client_address[0])
self.send_error(400)
return
if client_request == 'update':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.updates)
self._ctx.updates.clear()
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'node_info':
ip_addr = None
uid = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match(DispyAdminServer._NodeInfo.ip_re,
item.value):
ip_addr = item.value
else:
ip_addr = dispy._node_ipaddr(item.value)
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
node = self._ctx.nodes.get(ip_addr, None)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
if node:
node = dict(node.__dict__)
node.pop('_priv', None)
if node['avail_info']:
node['avail_info'] = node['avail_info'].__dict__
else:
node = {}
self.wfile.write(json.dumps(node).encode())
return
elif client_request == 'status':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.nodes)
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'get_uid':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
elif item.name == 'poll_interval':
try:
poll_interval = int(item.value)
assert poll_interval >= 5
except Exception:
self.send_error(400, 'invalid poll interval')
return
# TODO: only allow from http server?
uid = self._ctx.set_uid(self.client_address[0], poll_interval,
uid)
if not uid:
self.send_error(400, 'invalid uid')
return
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(uid).encode())
return
elif client_request == 'set_secret':
secret = None
uid = None
for item in form.list:
if item.name == 'secret':
secret = item.value.strip()
elif item.name == 'uid':
uid = item.value.strip()
if secret and uid == self._ctx.client_uid:
self._ctx.client_uid_time = time.time()
self._ctx.set_secret(secret)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'add_node':
host = ''
port = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'port':
try:
port = int(item.value)
except Exception:
port = None
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
if host and port:
ip_addr = dispy._node_ipaddr(host)
if ip_addr:
info = {'ip_addr': ip_addr, 'port': port}
Task(self._ctx.add_node, info)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'service_time':
hosts = []
svc_time = None
control = None
uid = None
for item in form.list:
if item.name == 'hosts':
hosts = [str(host) for host in json.loads(item.value)]
elif item.name == 'control':
control = item.value
elif item.name == 'time':
svc_time = item.value.strip()
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
for host in hosts:
Task(self._ctx.service_time, host, control, svc_time)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
return
elif client_request == 'set_cpus':
hosts = []
cpus = None
uid = None
for item in form.list:
if item.name == 'hosts':
hosts = [str(host) for host in json.loads(item.value)]
if not hosts:
self.send_error(400, 'invalid nodes')
return
elif item.name == 'cpus':
cpus = item.value
if cpus is not None:
try:
cpus = int(item.value)
except Exception:
self.send_error(400, 'invalid CPUs')
return
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
for host in hosts:
Task(self._ctx.set_cpus, host, cpus)
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
return
elif client_request == 'serve_clients':
host = ''
serve = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'serve':
serve = item.value
try:
serve = int(serve)
except Exception:
pass
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid and isinstance(serve, int)
and Task(self._ctx.serve_clients, host,
serve).value() == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
return
elif client_request == 'poll_interval':
uid = None
interval = None
for item in form.list:
if item.name == 'interval':
try:
interval = int(item.value)
except Exception:
if interval is not None:
logger.warning(
'%s: invalid poll interval "%s" ignored',
self._ctx.client_uid, item.value)
self.send_error(400)
return
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid
and self._ctx.set_poll_interval(interval) == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
logger.debug('Bad POST request from %s: %s',
self.client_address[0], client_request)
self.send_error(400)
return
def do_POST(self):
try:
form = cgi.FieldStorage(fp=self.rfile, headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
client_request = self.path[1:]
except Exception:
logger.debug('Ignoring invalid POST request from %s', self.client_address[0])
self.send_error(400)
return
if client_request == 'update':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.updates)
self._ctx.updates.clear()
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'node_info':
ip_addr = None
uid = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match(DispyAdminServer._NodeInfo.ip_re, item.value):
ip_addr = item.value
else:
ip_addr = dispy._node_ipaddr(item.value)
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
node = self._ctx.nodes.get(ip_addr, None)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
if node:
node = dict(node.__dict__)
node.pop('_priv', None)
if node['avail_info']:
node['avail_info'] = node['avail_info'].__dict__
else:
node = {}
self.wfile.write(json.dumps(node).encode())
return
elif client_request == 'status':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.nodes)
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'get_uid':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
now = time.time()
# TODO: only allow from http server?
if (self._ctx.client_uid and uid != self._ctx.client_uid and
((now - self._ctx.client_uid_time) < 3600)):
self.send_error(400, 'invalid uid')
return
if not uid:
uid = hashlib.sha1(os.urandom(20)).hexdigest()
self._ctx.client_uid = uid
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(uid).encode())
return
elif client_request == 'set_secret':
secret = None
uid = None
for item in form.list:
if item.name == 'secret':
secret = item.value.strip()
elif item.name == 'uid':
uid = item.value.strip()
if secret and uid == self._ctx.client_uid:
self._ctx.client_uid_time = time.time()
self._ctx.set_secret(secret)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'add_node':
host = ''
port = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'port':
try:
port = int(item.value)
except Exception:
port = None
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
if host and port:
ip_addr = dispy._node_ipaddr(host)
if ip_addr:
info = {'ip_addr': ip_addr, 'port': port}
Task(self._ctx.add_node, info)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'service_time':
hosts = []
svc_time = None
control = None
uid = None
for item in form.list:
if item.name == 'hosts':
hosts = [str(host) for host in json.loads(item.value)]
elif item.name == 'control':
control = item.value
elif item.name == 'time':
svc_time = item.value
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
for host in hosts:
Task(self._ctx.service_time, host, control, svc_time)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
return
elif client_request == 'set_cpus':
host = None
cpus = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value.strip()
elif item.name == 'cpus':
try:
cpus = int(item.value.strip())
except Exception:
pass
elif item.name == 'uid':
uid = item.value.strip()
if (host and cpus and uid == self._ctx.client_uid and
Task(self._ctx.set_cpus, host, cpus).value() == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'serve_clients':
host = ''
serve = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'serve':
serve = item.value
try:
serve = int(serve)
except Exception:
pass
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid and isinstance(serve, int) and
Task(self._ctx.serve_clients, host, serve).value() == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
return
elif client_request == 'poll_interval':
uid = None
interval = None
for item in form.list:
if item.name == 'interval':
try:
interval = int(item.value)
except Exception:
if interval is not None:
logger.warning('%s: invalid poll interval "%s" ignored',
self._ctx.client_uid, item.value)
self.send_error(400)
return
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid and self._ctx.set_poll_interval(interval) == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
logger.debug('Bad POST request from %s: %s', self.client_address[0], client_request)
self.send_error(400)
return
def do_GET(self):
if self.path == '/cluster_updates':
self._ctx._cluster_lock.acquire()
clusters = [{
'name':
name,
'jobs': {
'submitted': cluster.jobs_submitted,
'done': cluster.jobs_done
},
'nodes':
self.__class__.json_encode_nodes(cluster.updates)
} for name, cluster in dict_iter(self._ctx._clusters, 'items')]
for cluster in dict_iter(self._ctx._clusters, 'values'):
cluster.updates.clear()
self._ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(clusters).encode())
return
elif self.path == '/cluster_status':
self._ctx._cluster_lock.acquire()
clusters = [{
'name':
name,
'jobs': {
'submitted': cluster.jobs_submitted,
'done': cluster.jobs_done
},
'nodes':
self.__class__.json_encode_nodes(cluster.status)
} for name, cluster in dict_iter(self._ctx._clusters, 'items')]
self._ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(clusters).encode())
return
elif self.path == '/nodes':
self._ctx._cluster_lock.acquire()
nodes = [{
'name':
name,
'nodes':
self.__class__.json_encode_nodes(cluster.status)
} for name, cluster in dict_iter(self._ctx._clusters, 'items')]
self._ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
else:
parsed_path = urlparse(self.path)
path = parsed_path.path.lstrip('/')
if path == '' or path == 'index.html':
path = 'monitor.html'
path = os.path.join(self.DocumentRoot, path)
try:
with open(path) as fd:
data = fd.read()
if path.endswith('.html'):
if path.endswith('monitor.html') or path.endswith(
'node.html'):
data = data % {
'TIMEOUT':
str(self._ctx._poll_sec),
'SHOW_JOB_ARGS':
'true' if self._ctx._show_args else 'false'
}
content_type = 'text/html'
elif path.endswith('.js'):
content_type = 'text/javascript'
elif path.endswith('.css'):
content_type = 'text/css'
elif path.endswith('.ico'):
content_type = 'image/x-icon'
self.send_response(200)
self.send_header('Content-Type', content_type)
if content_type == 'text/css' or content_type == 'text/javascript':
self.send_header('Cache-Control',
'private, max-age=86400')
self.end_headers()
self.wfile.write(data.encode())
return
except Exception:
logger.warning('HTTP client %s: Could not read/send "%s"',
self.client_address[0], path)
logger.debug(traceback.format_exc())
self.send_error(404)
return
logger.debug('Bad GET request from %s: %s', self.client_address[0],
self.path)
self.send_error(400)
return
def do_POST(self):
try:
form = cgi.FieldStorage(fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
client_request = self.path[1:]
except Exception:
logger.debug('Ignoring invalid POST request from %s',
self.client_address[0])
self.send_error(400)
return
if client_request == 'node_info':
ip_addr = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match(DispyHTTPServer._ClusterInfo.ip_re,
item.value):
ip_addr = item.value
else:
ip_addr = dispy._node_ipaddr(item.value)
break
self._ctx._cluster_lock.acquire()
cluster_infos = [
(name, cluster_info)
for name, cluster_info in self._ctx._clusters.items()
]
self._ctx._cluster_lock.release()
cluster_jobs = {}
node = None
show_args = self._ctx._show_args
for name, cluster_info in cluster_infos:
cluster_node = cluster_info.status.get(ip_addr, None)
if not cluster_node:
cluster_jobs[name] = []
continue
if node:
node.jobs_done += cluster_node.jobs_done
node.cpu_time += cluster_node.cpu_time
node.update_time = max(node.update_time,
cluster_node.update_time)
node.tx += cluster_node.tx
node.rx += cluster_node.rx
else:
node = copy.copy(cluster_node)
# jobs = cluster_info.cluster.node_jobs(ip_addr)
jobs = [
job for job in dict_iter(cluster_info.jobs, 'values')
if job.ip_addr == ip_addr
]
# args and kwargs are sent as strings in Python,
# so an object's __str__ or __repr__ is used if provided;
# TODO: check job is in _ctx's jobs?
jobs = [{
'uid':
job._uid,
'job_id':
str(job.id),
'args':
', '.join(str(arg)
for arg in job._args) if show_args else '',
'kwargs':
', '.join('%s=%s' % (key, val)
for key, val in job._kwargs.items())
if show_args else '',
'start_time_ms':
int(1000 * job.start_time),
'cluster':
name
} for job in jobs]
cluster_jobs[name] = jobs
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
if node:
if node.avail_info:
node.avail_info = node.avail_info.__dict__
self.wfile.write(
json.dumps({
'node': node.__dict__,
'cluster_jobs': cluster_jobs
}).encode())
return
elif client_request == 'cancel_jobs':
uids = []
for item in form.list:
if item.name == 'uid':
try:
uids.append(int(item.value))
except ValueError:
logger.debug('Cancel job uid "%s" is invalid',
item.value)
self._ctx._cluster_lock.acquire()
cluster_jobs = [
(cluster_info.cluster, cluster_info.jobs.get(uid, None))
for cluster_info in self._ctx._clusters.values()
for uid in uids
]
self._ctx._cluster_lock.release()
cancelled = []
for cluster, job in cluster_jobs:
if not job:
continue
if cluster.cancel(job) == 0:
cancelled.append(job._uid)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(cancelled).encode())
return
elif client_request == 'add_node':
node = {'host': '', 'port': None, 'cpus': 0, 'cluster': None}
node_id = None
cluster = None
for item in form.list:
if item.name == 'host':
node['host'] = item.value
elif item.name == 'cluster':
node['cluster'] = item.value
elif item.name == 'port':
node['port'] = item.value
elif item.name == 'cpus':
try:
node['cpus'] = int(item.value)
except Exception:
pass
elif item.name == 'id':
node_id = item.value
if node['host']:
self._ctx._cluster_lock.acquire()
clusters = [
cluster_info.cluster
for name, cluster_info in self._ctx._clusters.items()
if name == node['cluster'] or not node['cluster']
]
self._ctx._cluster_lock.release()
for cluster in clusters:
cluster.allocate_node(node)
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
node['id'] = node_id
self.wfile.write(json.dumps(node).encode())
return
elif (client_request == 'close_node'
or client_request == 'allocate_node'
or client_request == 'deallocate_node'):
nodes = []
cluster_infos = []
resp = -1
for item in form.list:
if item.name == 'cluster':
self._ctx._cluster_lock.acquire()
if item.value == '*':
cluster_infos = list(self._ctx._clusters.values())
else:
cluster_infos = [
self._ctx._clusters.get(item.value, None)
]
if not cluster_infos[0]:
cluster_infos = []
self._ctx._cluster_lock.release()
elif item.name == 'nodes':
nodes = json.loads(item.value)
nodes = [str(node) for node in nodes]
if cluster_infos and nodes:
resp = 0
for cluster_info in cluster_infos:
fn = getattr(cluster_info.cluster, client_request)
if fn:
for node in nodes:
resp |= fn(node)
else:
resp = -1
self.send_response(200)
self.send_header(
'Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(resp).encode())
return
elif client_request == 'update':
for item in form.list:
if item.name == 'timeout':
try:
timeout = int(item.value)
if timeout < 1:
timeout = 0
self._ctx._poll_sec = timeout
except Exception:
logger.warning(
'HTTP client %s: invalid timeout "%s" ignored',
self.client_address[0], item.value)
elif item.name == 'show_job_args':
if item.value == 'true':
self._ctx._show_args = True
else:
self._ctx._show_args = False
return
elif client_request == 'set_cpus':
node_cpus = {}
for item in form.list:
self._ctx._cluster_lock.acquire()
for cluster_info in self._ctx._clusters.values():
node = cluster_info.status.get(item.name, None)
if node:
node_cpus[
item.
name] = cluster_info.cluster.set_node_cpus(
item.name, item.value)
if node_cpus[item.name] >= 0:
break
self._ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(node_cpus).encode())
return
logger.debug('Bad POST request from %s: %s',
self.client_address[0], client_request)
self.send_error(400)
return
def do_GET(self):
if self.path == '/cluster_updates':
self._dispy_ctx._cluster_lock.acquire()
clusters = [
{'name': name,
'jobs': {'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done},
'nodes': self.__class__.json_encode_nodes(cluster.updates)
} for name, cluster in dict_iter(self._dispy_ctx._clusters, 'items')
]
for cluster in dict_iter(self._dispy_ctx._clusters, 'values'):
cluster.updates.clear()
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(clusters).encode())
return
elif self.path == '/cluster_status':
self._dispy_ctx._cluster_lock.acquire()
clusters = [
{'name': name,
'jobs': {'submitted': cluster.jobs_submitted, 'done': cluster.jobs_done},
'nodes': self.__class__.json_encode_nodes(cluster.status)
} for name, cluster in dict_iter(self._dispy_ctx._clusters, 'items')
]
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(clusters).encode())
return
elif self.path == '/nodes':
self._dispy_ctx._cluster_lock.acquire()
nodes = [
{'name': name,
'nodes': self.__class__.json_encode_nodes(cluster.status)
} for name, cluster in dict_iter(self._dispy_ctx._clusters, 'items')
]
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
else:
parsed_path = urlparse(self.path)
path = parsed_path.path.lstrip('/')
if path == '' or path == 'index.html':
path = 'monitor.html'
path = os.path.join(self.DocumentRoot, path)
try:
with open(path) as fd:
data = fd.read()
if path.endswith('.html'):
if path.endswith('monitor.html') or path.endswith('node.html'):
data = data % {'TIMEOUT': str(self._dispy_ctx._poll_sec)}
content_type = 'text/html'
elif path.endswith('.js'):
content_type = 'text/javascript'
elif path.endswith('.css'):
content_type = 'text/css'
elif path.endswith('.ico'):
content_type = 'image/x-icon'
self.send_response(200)
self.send_header('Content-Type', content_type)
if content_type == 'text/css' or content_type == 'text/javascript':
self.send_header('Cache-Control', 'private, max-age=86400')
self.end_headers()
self.wfile.write(data.encode())
return
except:
logger.warning('HTTP client %s: Could not read/send "%s"',
self.client_address[0], path)
logger.debug(traceback.format_exc())
self.send_error(404)
return
logger.debug('Bad GET request from %s: %s' % (self.client_address[0], self.path))
self.send_error(400)
return
def do_POST(self):
form = cgi.FieldStorage(fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
if self.path == '/node_jobs':
ip_addr = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match('^\d+[\.\d]+$', item.value):
ip_addr = item.value
else:
try:
ip_addr = socket.gethostbyname(item.value)
except:
ip_addr = item.value
break
self._dispy_ctx._cluster_lock.acquire()
cluster_infos = [(name, cluster_info) for name, cluster_info in
self._dispy_ctx._clusters.items()]
self._dispy_ctx._cluster_lock.release()
jobs = []
node = None
for name, cluster_info in cluster_infos:
cluster_node = cluster_info.status.get(ip_addr, None)
if not cluster_node:
continue
if node:
node.jobs_done += cluster_node.jobs_done
node.cpu_time += cluster_node.cpu_time
node.update_time = max(node.update_time,
cluster_node.update_time)
else:
node = copy.copy(cluster_node)
cluster_jobs = cluster_info.cluster.node_jobs(ip_addr)
# args and kwargs are sent as strings in Python,
# so an object's __str__ or __repr__ is used if provided;
# TODO: check job is in _dispy_ctx's jobs?
jobs.extend([{
'uid':
id(job),
'job_id':
str(job.id),
'args':
', '.join(str(arg) for arg in job.args),
'kwargs':
', '.join('%s=%s' % (key, val)
for key, val in job.kwargs.items()),
'sched_time_ms':
int(1000 * job.start_time),
'cluster':
name
} for job in cluster_jobs])
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
if node and node.avail_info:
node.avail_info = node.avail_info.__dict__
self.wfile.write(
json.dumps({
'node': node.__dict__,
'jobs': jobs
}).encode())
return
elif self.path == '/cancel_jobs':
uids = []
for item in form.list:
if item.name == 'uid':
try:
uids.append(int(item.value))
except ValueError:
logger.debug('Cancel job uid "%s" is invalid',
item.value)
self._dispy_ctx._cluster_lock.acquire()
cluster_jobs = [
(cluster_info.cluster, cluster_info.jobs.get(uid, None))
for cluster_info in self._dispy_ctx._clusters.values()
for uid in uids
]
self._dispy_ctx._cluster_lock.release()
cancelled = []
for cluster, job in cluster_jobs:
if not job:
continue
if cluster.cancel(job) == 0:
cancelled.append(id(job))
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(cancelled).encode())
return
elif self.path == '/terminate_cluster':
names_of_clusters = [i.value for i in form.list]
del_jobs = [
(cluster_info, job_id)
for cluster_info in self._dispy_ctx._clusters.values()
if cluster_info.cluster.name in names_of_clusters
for job_id in cluster_info.jobs.keys()
]
ret_values = [
cluster_info.cluster.cancel(cluster_info.jobs[job_id])
for cluster_info, job_id in del_jobs
]
delete_clusters = [
cluster_info.cluster
for cluster_info in self._dispy_ctx._clusters.values()
if cluster_info.cluster.name in names_of_clusters
]
[self._dispy_ctx.del_cluster(cl) for cl in delete_clusters]
if sum(ret_values) == 0:
self.send_response(200)
self.send_header('Content-Type',
'text/plain; charset=utf-8')
self.end_headers()
self.wfile.write("OK".encode())
return
elif self.path == '/add_node':
node = {'host': '', 'port': None, 'cpus': 0, 'cluster': None}
node_id = None
cluster = None
for item in form.list:
if item.name == 'host':
node['host'] = item.value
elif item.name == 'cluster':
node['cluster'] = item.value
elif item.name == 'port':
node['port'] = item.value
elif item.name == 'cpus':
try:
node['cpus'] = int(item.value)
except:
pass
elif item.name == 'id':
node_id = item.value
if node['host']:
self._dispy_ctx._cluster_lock.acquire()
clusters = [
cluster_info.cluster for name, cluster_info in
self._dispy_ctx._clusters.items()
if name == node['cluster'] or not node['cluster']
]
self._dispy_ctx._cluster_lock.release()
for cluster in clusters:
cluster.allocate_node(node)
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
node['id'] = node_id
self.wfile.write(json.dumps(node).encode())
return
elif self.path == '/set_poll_sec':
for item in form.list:
if item.name != 'timeout':
continue
try:
timeout = int(item.value)
if timeout < 1:
timeout = 0
except:
logger.warning(
'HTTP client %s: invalid timeout "%s" ignored',
self.client_address[0], item.value)
timeout = 0
self._dispy_ctx._poll_sec = timeout
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
return
elif self.path == '/set_cpus':
node_cpus = {}
for item in form.list:
self._dispy_ctx._cluster_lock.acquire()
for cluster_info in self._dispy_ctx._clusters.values():
node = cluster_info.status.get(item.name, None)
if node:
node_cpus[
item.
name] = cluster_info.cluster.set_node_cpus(
item.name, item.value)
if node_cpus[item.name] >= 0:
break
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(node_cpus).encode())
return
logger.debug('Bad POST request from %s: %s',
self.client_address[0], self.path)
self.send_error(400)
return
def udp_server(self, addrinfo, task=None):
task.set_daemon()
udp_sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_DGRAM))
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception:
pass
udp_sock.bind((addrinfo.bind_addr, self.info_port))
if addrinfo.family == socket.AF_INET:
if self.ipv4_udp_multicast:
mreq = socket.inet_aton(addrinfo.broadcast) + socket.inet_aton(addrinfo.ip)
udp_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
else: # addrinfo.family == socket.AF_INET6:
mreq = socket.inet_pton(addrinfo.family, addrinfo.broadcast)
mreq += struct.pack('@I', addrinfo.ifn)
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
try:
udp_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
except Exception:
pass
while 1:
msg, addr = yield udp_sock.recvfrom(1000)
if msg.startswith('PING:'):
try:
info = deserialize(msg[len('PING:'):])
if info['version'] != _dispy_version:
logger.warning('Ignoring %s due to version mismatch', addr[0])
continue
assert info['port'] > 0
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node:
if node._priv.sign == info['sign']:
Task(self.update_node_info, node)
else:
node._priv.sign = info['sign']
node._priv.auth = None
Task(self.get_node_info, node)
else:
info['family'] = addrinfo.family
Task(self.add_node, info)
elif msg.startswith('TERMINATED:'):
try:
info = deserialize(msg[len('TERMINATED:'):])
assert info['ip_addr']
except Exception:
logger.debug('Ignoring node %s', addr[0])
continue
node = self.nodes.get(info['ip_addr'], None)
if node and node._priv.sign == info['sign']:
with self.lock:
self.nodes.pop(info['ip_addr'], None)
def do_POST(self):
form = cgi.FieldStorage(fp=self.rfile, headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
if self.path == '/node_jobs':
ip_addr = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match('^\d+[\.\d]+$', item.value):
ip_addr = item.value
else:
try:
ip_addr = socket.gethostbyname(item.value)
except:
ip_addr = item.value
break
self._dispy_ctx._cluster_lock.acquire()
cluster_infos = [(name, cluster_info) for name, cluster_info in
self._dispy_ctx._clusters.items()]
self._dispy_ctx._cluster_lock.release()
jobs = []
node = None
for name, cluster_info in cluster_infos:
cluster_node = cluster_info.status.get(ip_addr, None)
if not cluster_node:
continue
if node:
node.jobs_done += cluster_node.jobs_done
node.cpu_time += cluster_node.cpu_time
node.update_time = max(node.update_time, cluster_node.update_time)
else:
node = copy.copy(cluster_node)
cluster_jobs = cluster_info.cluster.node_jobs(ip_addr)
# args and kwargs are sent as strings in Python,
# so an object's __str__ or __repr__ is used if provided;
# TODO: check job is in _dispy_ctx's jobs?
jobs.extend([{'uid': id(job), 'job_id': str(job.id),
'args': ', '.join(str(arg) for arg in job.args),
'kwargs': ', '.join('%s=%s' % (key, val)
for key, val in job.kwargs.items()),
'sched_time_ms': int(1000 * job.start_time),
'cluster': name}
for job in cluster_jobs])
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
if node and node.avail_info:
node.avail_info = node.avail_info.__dict__
self.wfile.write(json.dumps({'node': node.__dict__, 'jobs': jobs}).encode())
return
elif self.path == '/cancel_jobs':
uids = []
for item in form.list:
if item.name == 'uid':
try:
uids.append(int(item.value))
except ValueError:
logger.debug('Cancel job uid "%s" is invalid' % item.value)
self._dispy_ctx._cluster_lock.acquire()
cluster_jobs = [(cluster_info.cluster, cluster_info.jobs.get(uid, None))
for cluster_info in self._dispy_ctx._clusters.values()
for uid in uids]
self._dispy_ctx._cluster_lock.release()
cancelled = []
for cluster, job in cluster_jobs:
if not job:
continue
if cluster.cancel(job) == 0:
cancelled.append(id(job))
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(cancelled).encode())
return
elif self.path == '/add_node':
node = {'host': '', 'port': None, 'cpus': 0, 'cluster': None}
node_id = None
cluster = None
for item in form.list:
if item.name == 'host':
node['host'] = item.value
elif item.name == 'cluster':
node['cluster'] = item.value
elif item.name == 'port':
node['port'] = item.value
elif item.name == 'cpus':
try:
node['cpus'] = int(item.value)
except:
pass
elif item.name == 'id':
node_id = item.value
if node['host']:
self._dispy_ctx._cluster_lock.acquire()
clusters = [cluster_info.cluster for name, cluster_info in
self._dispy_ctx._clusters.items()
if name == node['cluster'] or not node['cluster']]
self._dispy_ctx._cluster_lock.release()
for cluster in clusters:
cluster.allocate_node(node)
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
node['id'] = node_id
self.wfile.write(json.dumps(node).encode())
return
elif self.path == '/set_poll_sec':
for item in form.list:
if item.name != 'timeout':
continue
try:
timeout = int(item.value)
if timeout < 1:
timeout = 0
except:
logger.warning('HTTP client %s: invalid timeout "%s" ignored',
self.client_address[0], item.value)
timeout = 0
self._dispy_ctx._poll_sec = timeout
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
return
elif self.path == '/set_cpus':
node_cpus = {}
for item in form.list:
self._dispy_ctx._cluster_lock.acquire()
for cluster_info in self._dispy_ctx._clusters.values():
node = cluster_info.status.get(item.name, None)
if node:
node_cpus[item.name] = cluster_info.cluster.set_node_cpus(
item.name, item.value)
if node_cpus[item.name] >= 0:
break
self._dispy_ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(node_cpus).encode())
return
logger.debug('Bad POST request from %s: %s' % (self.client_address[0], self.path))
self.send_error(400)
return
