def serve_clients(self, host, serve, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVE_CLIENTS:'.encode() +
serialize({'serve': serve}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.serve = info['serve']
resp = 0
except Exception:
dispy.logger.debug('Setting serve clients %s to %s failed', host,
serve)
resp = -1
finally:
sock.close()
raise StopIteration(resp)
def __init__(self,
ip_addrs=[],
node_port=51348,
listen_port=0,
scheduler_node=None,
scheduler_port=51347):
addrinfos = []
if not ip_addrs:
ip_addrs = [None]
for i in range(len(ip_addrs)):
ip_addr = ip_addrs[i]
addrinfo = dispy.node_addrinfo(ip_addr)
if not addrinfo:
logger.warning('Ignoring invalid ip_addr %s', ip_addr)
continue
addrinfos.append(addrinfo)
if not listen_port:
listen_port = node_port
self.node_port = node_port
self.listen_port = listen_port
self.scheduler_port = scheduler_port
self.scheduler_ip_addrs = list(
filter(lambda ip: bool(ip), [dispy._node_ipaddr(scheduler_node)]))
for addrinfo in addrinfos:
self.listen_udp_task = Task(self.listen_udp_proc, addrinfo)
self.listen_tcp_task = Task(self.listen_tcp_proc, addrinfo)
self.sched_udp_task = Task(self.sched_udp_proc, addrinfo)
logger.info('version %s started', dispy._dispy_version)
def service_time(self, host, control, time, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family,
socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVICE_TIME:'.encode() +
serialize({
'control': control,
'time': time
}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.service_start = info['service_start']
node.service_stop = info['service_stop']
node.service_end = info['service_end']
resp = 0
except Exception:
resp = -1
sock.close()
if resp:
dispy.logger.debug('Setting service %s time of %s to %s failed',
control, host, time)
raise StopIteration(resp)
def __init__(self, node_port=51348, listen_port=0, scheduler_node=None, scheduler_port=51347):
if not listen_port:
listen_port = node_port
self.node_port = node_port
self.listen_port = listen_port
self.scheduler_port = scheduler_port
self.scheduler_ip_addrs = list(filter(lambda ip: bool(ip), [dispy._node_ipaddr(scheduler_node)]))
self.listen_udp_coro = asyncoro.Coro(self.listen_udp_proc)
self.listen_tcp_coro = asyncoro.Coro(self.listen_tcp_proc)
self.sched_udp_coro = asyncoro.Coro(self.sched_udp_proc)
def __init__(self, ip_addrs=[], node_port=51348, relay_port=0,
scheduler_nodes=[], scheduler_port=51347, ipv4_udp_multicast=False,
secret='', certfile=None, keyfile=None):
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
addrinfos = []
if not ip_addrs:
ip_addrs = [None]
for i in range(len(ip_addrs)):
ip_addr = ip_addrs[i]
addrinfo = dispy.host_addrinfo(host=ip_addr, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid ip_addr %s', ip_addr)
continue
addrinfos.append(addrinfo)
self.node_port = node_port
if not relay_port:
relay_port = node_port
self.relay_port = relay_port
self.ip_addrs = set()
self.scheduler_ip_addr = None
self.scheduler_port = scheduler_port
self.secret = secret
if certfile:
self.certfile = os.path.abspath(certfile)
else:
self.certfile = None
if keyfile:
self.keyfile = os.path.abspath(keyfile)
else:
self.keyfile = None
udp_addrinfos = {}
for addrinfo in addrinfos:
self.ip_addrs.add(addrinfo.ip)
Task(self.relay_tcp_proc, addrinfo)
udp_addrinfos[addrinfo.bind_addr] = addrinfo
scheduler_ip_addrs = []
for addr in scheduler_nodes:
addr = dispy._node_ipaddr(addr)
if addr:
scheduler_ip_addrs.append(addr)
for bind_addr, addrinfo in udp_addrinfos.items():
Task(self.relay_udp_proc, bind_addr, addrinfo)
Task(self.sched_udp_proc, bind_addr, addrinfo)
for addr in scheduler_ip_addrs:
msg = {'version': __version__, 'ip_addrs': [addr], 'port': self.scheduler_port,
'sign': None}
Task(self.verify_broadcast, addrinfo, msg)
logger.info('version %s started', dispy._dispy_version)
def __init__(self, ip_addrs=[], relay_port=0, scheduler_nodes=[], scheduler_port=0,
ipv4_udp_multicast=False, secret='', certfile=None, keyfile=None):
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
addrinfos = []
if not ip_addrs:
ip_addrs = [None]
for i in range(len(ip_addrs)):
ip_addr = ip_addrs[i]
addrinfo = dispy.host_addrinfo(host=ip_addr, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid ip_addr %s', ip_addr)
continue
addrinfos.append(addrinfo)
self.node_port = eval(dispy.config.NodePort)
self.scheduler_port = scheduler_port
self.relay_port = relay_port
self.ip_addrs = set()
self.scheduler_ip_addr = None
self.secret = secret
if certfile:
self.certfile = os.path.abspath(certfile)
else:
self.certfile = None
if keyfile:
self.keyfile = os.path.abspath(keyfile)
else:
self.keyfile = None
udp_addrinfos = {}
for addrinfo in addrinfos:
self.ip_addrs.add(addrinfo.ip)
Task(self.relay_tcp_proc, addrinfo)
udp_addrinfos[addrinfo.bind_addr] = addrinfo
scheduler_ip_addrs = []
for addr in scheduler_nodes:
addr = dispy._node_ipaddr(addr)
if addr:
scheduler_ip_addrs.append(addr)
for bind_addr, addrinfo in udp_addrinfos.items():
Task(self.relay_udp_proc, bind_addr, addrinfo)
Task(self.sched_udp_proc, bind_addr, addrinfo)
for addr in scheduler_ip_addrs:
msg = {'version': __version__, 'ip_addrs': [addr], 'port': self.scheduler_port,
'sign': None}
Task(self.verify_broadcast, addrinfo, msg)
logger.info('version %s started', dispy._dispy_version)
def __init__(self,
node_port=51348,
listen_port=0,
scheduler_node=None,
scheduler_port=51347):
if not listen_port:
listen_port = node_port
self.node_port = node_port
self.listen_port = listen_port
self.scheduler_port = scheduler_port
self.scheduler_ip_addrs = list(
filter(lambda ip: bool(ip), [dispy._node_ipaddr(scheduler_node)]))
self.listen_udp_coro = asyncoro.Coro(self.listen_udp_proc)
self.listen_tcp_coro = asyncoro.Coro(self.listen_tcp_proc)
self.sched_udp_coro = asyncoro.Coro(self.sched_udp_proc)
def serve_clients(self, host, serve, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVE_CLIENTS:' + serialize({'serve': serve}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.serve = info['serve']
resp = 0
except Exception:
dispy.logger.debug('Setting serve %s to %s failed', host, serve)
resp = -1
finally:
sock.close()
raise StopIteration(resp)
def service_time(self, host, control, time, task=None):
node = self.nodes.get(dispy._node_ipaddr(host), None)
if not node or not node._priv.auth:
raise StopIteration(-1)
sock = AsyncSocket(socket.socket(node._priv.sock_family, socket.SOCK_STREAM),
keyfile=self.keyfile, certfile=self.certfile)
sock.settimeout(MsgTimeout)
try:
yield sock.connect((node.ip_addr, node._priv.port))
yield sock.sendall(node._priv.auth)
yield sock.send_msg('SERVICE_TIME:' + serialize({'control': control, 'time': time}))
resp = yield sock.recv_msg()
info = deserialize(resp)
node.service_start = info['service_start']
node.service_stop = info['service_stop']
node.service_end = info['service_end']
resp = 0
except Exception:
resp = -1
sock.close()
if resp:
dispy.logger.debug('Setting service %s time of %s to %s failed', control, host, time)
raise StopIteration(resp)
def __init__(self,
ip_addr=None,
node_port=51348,
listen_port=0,
scheduler_node=None,
scheduler_port=51347):
self.addrinfo = dispy.node_addrinfo(ip_addr)
if not listen_port:
listen_port = node_port
self.node_port = node_port
self.listen_port = listen_port
self.scheduler_port = scheduler_port
self.scheduler_ip_addrs = list(
filter(lambda ip: bool(ip), [dispy._node_ipaddr(scheduler_node)]))
self.listen_udp_coro = asyncoro.Coro(self.listen_udp_proc)
self.listen_tcp_coro = asyncoro.Coro(self.listen_tcp_proc)
self.sched_udp_coro = asyncoro.Coro(self.sched_udp_proc)
if self.addrinfo[0] == socket.AF_INET:
self._broadcast = '<broadcast>'
if netifaces:
for iface in netifaces.interfaces():
for link in netifaces.ifaddresses(iface).get(
netifaces.AF_INET, []):
if link['addr'] == self.addrinfo[4][0]:
self._broadcast = link.get('broadcast',
'<broadcast>')
break
else:
continue
break
else: # self.sock_family == socket.AF_INET6
self._broadcast = 'ff02::1'
addrinfo = socket.getaddrinfo(self._broadcast, None)[0]
self.mreq = socket.inet_pton(addrinfo[0], addrinfo[4][0])
self.mreq += struct.pack('@I', self.addrinfo[4][-1])
logger.info('version %s started', dispy._dispy_version)
def do_POST(self):
try:
form = cgi.FieldStorage(fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
client_request = self.path[1:]
except Exception:
logger.debug('Ignoring invalid POST request from %s',
self.client_address[0])
self.send_error(400)
return
if client_request == 'update':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.updates)
self._ctx.updates.clear()
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'node_info':
ip_addr = None
uid = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match(DispyAdminServer._NodeInfo.ip_re,
item.value):
ip_addr = item.value
else:
ip_addr = dispy._node_ipaddr(item.value)
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
node = self._ctx.nodes.get(ip_addr, None)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
if node:
node = dict(node.__dict__)
node.pop('_priv', None)
if node['avail_info']:
node['avail_info'] = node['avail_info'].__dict__
else:
node = {}
self.wfile.write(json.dumps(node).encode())
return
elif client_request == 'status':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.nodes)
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'get_uid':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
elif item.name == 'poll_interval':
try:
poll_interval = int(item.value)
assert poll_interval >= 5
except Exception:
self.send_error(400, 'invalid poll interval')
return
# TODO: only allow from http server?
uid = self._ctx.set_uid(self.client_address[0], poll_interval,
uid)
if not uid:
self.send_error(400, 'invalid uid')
return
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(uid).encode())
return
elif client_request == 'set_secret':
secret = None
uid = None
for item in form.list:
if item.name == 'secret':
secret = item.value.strip()
elif item.name == 'uid':
uid = item.value.strip()
if secret and uid == self._ctx.client_uid:
self._ctx.client_uid_time = time.time()
self._ctx.set_secret(secret)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'add_node':
host = ''
port = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'port':
try:
port = int(item.value)
except Exception:
port = None
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
if host and port:
ip_addr = dispy._node_ipaddr(host)
if ip_addr:
info = {'ip_addr': ip_addr, 'port': port}
Task(self._ctx.add_node, info)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'service_time':
hosts = []
svc_time = None
control = None
uid = None
for item in form.list:
if item.name == 'hosts':
hosts = [str(host) for host in json.loads(item.value)]
elif item.name == 'control':
control = item.value
elif item.name == 'time':
svc_time = item.value.strip()
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
for host in hosts:
Task(self._ctx.service_time, host, control, svc_time)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
return
elif client_request == 'set_cpus':
hosts = []
cpus = None
uid = None
for item in form.list:
if item.name == 'hosts':
hosts = [str(host) for host in json.loads(item.value)]
if not hosts:
self.send_error(400, 'invalid nodes')
return
elif item.name == 'cpus':
cpus = item.value
if cpus is not None:
try:
cpus = int(item.value)
except Exception:
self.send_error(400, 'invalid CPUs')
return
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
for host in hosts:
Task(self._ctx.set_cpus, host, cpus)
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
return
elif client_request == 'serve_clients':
host = ''
serve = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'serve':
serve = item.value
try:
serve = int(serve)
except Exception:
pass
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid and isinstance(serve, int)
and Task(self._ctx.serve_clients, host,
serve).value() == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
return
elif client_request == 'poll_interval':
uid = None
interval = None
for item in form.list:
if item.name == 'interval':
try:
interval = int(item.value)
except Exception:
if interval is not None:
logger.warning(
'%s: invalid poll interval "%s" ignored',
self._ctx.client_uid, item.value)
self.send_error(400)
return
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid
and self._ctx.set_poll_interval(interval) == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
logger.debug('Bad POST request from %s: %s',
self.client_address[0], client_request)
self.send_error(400)
return
def do_POST(self):
try:
form = cgi.FieldStorage(fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
client_request = self.path[1:]
except Exception:
logger.debug('Ignoring invalid POST request from %s',
self.client_address[0])
self.send_error(400)
return
if client_request == 'node_info':
ip_addr = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match(DispyHTTPServer._ClusterInfo.ip_re,
item.value):
ip_addr = item.value
else:
ip_addr = dispy._node_ipaddr(item.value)
break
self._ctx._cluster_lock.acquire()
cluster_infos = [
(name, cluster_info)
for name, cluster_info in self._ctx._clusters.items()
]
self._ctx._cluster_lock.release()
cluster_jobs = {}
node = None
show_args = self._ctx._show_args
for name, cluster_info in cluster_infos:
cluster_node = cluster_info.status.get(ip_addr, None)
if not cluster_node:
cluster_jobs[name] = []
continue
if node:
node.jobs_done += cluster_node.jobs_done
node.cpu_time += cluster_node.cpu_time
node.update_time = max(node.update_time,
cluster_node.update_time)
node.tx += cluster_node.tx
node.rx += cluster_node.rx
else:
node = copy.copy(cluster_node)
# jobs = cluster_info.cluster.node_jobs(ip_addr)
jobs = [
job for job in dict_iter(cluster_info.jobs, 'values')
if job.ip_addr == ip_addr
]
# args and kwargs are sent as strings in Python,
# so an object's __str__ or __repr__ is used if provided;
# TODO: check job is in _ctx's jobs?
jobs = [{
'uid':
job._uid,
'job_id':
str(job.id),
'args':
', '.join(str(arg)
for arg in job._args) if show_args else '',
'kwargs':
', '.join('%s=%s' % (key, val)
for key, val in job._kwargs.items())
if show_args else '',
'start_time_ms':
int(1000 * job.start_time),
'cluster':
name
} for job in jobs]
cluster_jobs[name] = jobs
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
if node:
if node.avail_info:
node.avail_info = node.avail_info.__dict__
self.wfile.write(
json.dumps({
'node': node.__dict__,
'cluster_jobs': cluster_jobs
}).encode())
return
elif client_request == 'cancel_jobs':
uids = []
for item in form.list:
if item.name == 'uid':
try:
uids.append(int(item.value))
except ValueError:
logger.debug('Cancel job uid "%s" is invalid',
item.value)
self._ctx._cluster_lock.acquire()
cluster_jobs = [
(cluster_info.cluster, cluster_info.jobs.get(uid, None))
for cluster_info in self._ctx._clusters.values()
for uid in uids
]
self._ctx._cluster_lock.release()
cancelled = []
for cluster, job in cluster_jobs:
if not job:
continue
if cluster.cancel(job) == 0:
cancelled.append(job._uid)
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(cancelled).encode())
return
elif client_request == 'add_node':
node = {'host': '', 'port': None, 'cpus': 0, 'cluster': None}
node_id = None
cluster = None
for item in form.list:
if item.name == 'host':
node['host'] = item.value
elif item.name == 'cluster':
node['cluster'] = item.value
elif item.name == 'port':
node['port'] = item.value
elif item.name == 'cpus':
try:
node['cpus'] = int(item.value)
except Exception:
pass
elif item.name == 'id':
node_id = item.value
if node['host']:
self._ctx._cluster_lock.acquire()
clusters = [
cluster_info.cluster
for name, cluster_info in self._ctx._clusters.items()
if name == node['cluster'] or not node['cluster']
]
self._ctx._cluster_lock.release()
for cluster in clusters:
cluster.allocate_node(node)
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
node['id'] = node_id
self.wfile.write(json.dumps(node).encode())
return
elif (client_request == 'close_node'
or client_request == 'allocate_node'
or client_request == 'deallocate_node'):
nodes = []
cluster_infos = []
resp = -1
for item in form.list:
if item.name == 'cluster':
self._ctx._cluster_lock.acquire()
if item.value == '*':
cluster_infos = list(self._ctx._clusters.values())
else:
cluster_infos = [
self._ctx._clusters.get(item.value, None)
]
if not cluster_infos[0]:
cluster_infos = []
self._ctx._cluster_lock.release()
elif item.name == 'nodes':
nodes = json.loads(item.value)
nodes = [str(node) for node in nodes]
if cluster_infos and nodes:
resp = 0
for cluster_info in cluster_infos:
fn = getattr(cluster_info.cluster, client_request)
if fn:
for node in nodes:
resp |= fn(node)
else:
resp = -1
self.send_response(200)
self.send_header(
'Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(resp).encode())
return
elif client_request == 'update':
for item in form.list:
if item.name == 'timeout':
try:
timeout = int(item.value)
if timeout < 1:
timeout = 0
self._ctx._poll_sec = timeout
except Exception:
logger.warning(
'HTTP client %s: invalid timeout "%s" ignored',
self.client_address[0], item.value)
elif item.name == 'show_job_args':
if item.value == 'true':
self._ctx._show_args = True
else:
self._ctx._show_args = False
return
elif client_request == 'set_cpus':
node_cpus = {}
for item in form.list:
self._ctx._cluster_lock.acquire()
for cluster_info in self._ctx._clusters.values():
node = cluster_info.status.get(item.name, None)
if node:
node_cpus[
item.
name] = cluster_info.cluster.set_node_cpus(
item.name, item.value)
if node_cpus[item.name] >= 0:
break
self._ctx._cluster_lock.release()
self.send_response(200)
self.send_header('Content-Type',
'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(node_cpus).encode())
return
logger.debug('Bad POST request from %s: %s',
self.client_address[0], client_request)
self.send_error(400)
return
def do_POST(self):
try:
form = cgi.FieldStorage(fp=self.rfile, headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
client_request = self.path[1:]
except Exception:
logger.debug('Ignoring invalid POST request from %s', self.client_address[0])
self.send_error(400)
return
if client_request == 'update':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.updates)
self._ctx.updates.clear()
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'node_info':
ip_addr = None
uid = None
for item in form.list:
if item.name == 'host':
# if it looks like IP address, skip resolving
if re.match(DispyAdminServer._NodeInfo.ip_re, item.value):
ip_addr = item.value
else:
ip_addr = dispy._node_ipaddr(item.value)
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
node = self._ctx.nodes.get(ip_addr, None)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
if node:
node = dict(node.__dict__)
node.pop('_priv', None)
if node['avail_info']:
node['avail_info'] = node['avail_info'].__dict__
else:
node = {}
self.wfile.write(json.dumps(node).encode())
return
elif client_request == 'status':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
self._ctx.lock.acquire()
nodes = self.__class__.json_encode_nodes(self._ctx.nodes)
self._ctx.lock.release()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(nodes).encode())
return
elif client_request == 'get_uid':
uid = None
for item in form.list:
if item.name == 'uid':
uid = item.value.strip()
break
now = time.time()
# TODO: only allow from http server?
if (self._ctx.client_uid and uid != self._ctx.client_uid and
((now - self._ctx.client_uid_time) < 3600)):
self.send_error(400, 'invalid uid')
return
if not uid:
uid = hashlib.sha1(os.urandom(20)).hexdigest()
self._ctx.client_uid = uid
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(uid).encode())
return
elif client_request == 'set_secret':
secret = None
uid = None
for item in form.list:
if item.name == 'secret':
secret = item.value.strip()
elif item.name == 'uid':
uid = item.value.strip()
if secret and uid == self._ctx.client_uid:
self._ctx.client_uid_time = time.time()
self._ctx.set_secret(secret)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'add_node':
host = ''
port = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'port':
try:
port = int(item.value)
except Exception:
port = None
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
if host and port:
ip_addr = dispy._node_ipaddr(host)
if ip_addr:
info = {'ip_addr': ip_addr, 'port': port}
Task(self._ctx.add_node, info)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'service_time':
hosts = []
svc_time = None
control = None
uid = None
for item in form.list:
if item.name == 'hosts':
hosts = [str(host) for host in json.loads(item.value)]
elif item.name == 'control':
control = item.value
elif item.name == 'time':
svc_time = item.value
elif item.name == 'uid':
uid = item.value.strip()
if uid != self._ctx.client_uid:
self.send_error(400, 'invalid uid')
return
self._ctx.client_uid_time = time.time()
for host in hosts:
Task(self._ctx.service_time, host, control, svc_time)
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
return
elif client_request == 'set_cpus':
host = None
cpus = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value.strip()
elif item.name == 'cpus':
try:
cpus = int(item.value.strip())
except Exception:
pass
elif item.name == 'uid':
uid = item.value.strip()
if (host and cpus and uid == self._ctx.client_uid and
Task(self._ctx.set_cpus, host, cpus).value() == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
elif client_request == 'serve_clients':
host = ''
serve = None
uid = None
for item in form.list:
if item.name == 'host':
host = item.value
elif item.name == 'serve':
serve = item.value
try:
serve = int(serve)
except Exception:
pass
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid and isinstance(serve, int) and
Task(self._ctx.serve_clients, host, serve).value() == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
return
elif client_request == 'poll_interval':
uid = None
interval = None
for item in form.list:
if item.name == 'interval':
try:
interval = int(item.value)
except Exception:
if interval is not None:
logger.warning('%s: invalid poll interval "%s" ignored',
self._ctx.client_uid, item.value)
self.send_error(400)
return
elif item.name == 'uid':
uid = item.value.strip()
if (uid == self._ctx.client_uid and self._ctx.set_poll_interval(interval) == 0):
self._ctx.client_uid_time = time.time()
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(0).encode())
else:
self.send_error(400)
return
logger.debug('Bad POST request from %s: %s', self.client_address[0], client_request)
self.send_error(400)
return
def relay_pings(self, ip_addr='', netmask=None, node_port=51348,
scheduler_node=None, scheduler_port=51347):
netaddr = None
if not netmask:
try:
ip_addr, bits = ip_addr.split('/')
socket.inet_aton(ip_addr)
netmask = (0xffffffff << (32 - int(bits))) & 0xffffffff
netaddr = (struct.unpack('>L', socket.inet_aton(ip_addr))[0]) & netmask
except:
netmask = '255.255.255.255'
if ip_addr:
socket.inet_aton(ip_addr)
else:
ip_addr = socket.gethostbyname(socket.gethostname())
if not netaddr and netmask:
try:
if isinstance(netmask, str):
netmask = struct.unpack('>L', socket.inet_aton(netmask))[0]
else:
assert isinstance(netmask, int)
assert netmask > 0
netaddr = (struct.unpack('>L', socket.inet_aton(ip_addr))[0]) & netmask
except:
logger.warning('Invalid netmask')
try:
socket.inet_ntoa(struct.pack('>L', netaddr))
socket.inet_ntoa(struct.pack('>L', netmask))
except:
netaddr = netmask = None
scheduler_version = _dispy_version
bc_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bc_sock.bind(('', 0))
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
scheduler_ip_addr = _node_ipaddr(scheduler_node)
if scheduler_ip_addr and scheduler_port:
relay_request = serialize({'ip_addr':scheduler_ip_addr, 'port':scheduler_port,
'version':_dispy_version, 'sign':None})
bc_sock.sendto('PING:%s' % relay_request, ('<broadcast>', node_port))
node_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
node_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
node_sock.bind(('', node_port))
sched_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sched_sock.bind(('', scheduler_port))
logger.info('Listening on %s:%s/%s', ip_addr, node_port, scheduler_port)
while True:
ready = select.select([node_sock, sched_sock], [], [])[0]
for sock in ready:
if sock == node_sock:
msg, addr = node_sock.recvfrom(1024)
if not msg.startswith('PING:'):
logger.debug('Ignoring message "%s" from %s',
msg[:min(len(msg), 5)], addr[0])
continue
if netaddr and (struct.unpack('>L', socket.inet_aton(addr[0]))[0] & netmask) == netaddr:
logger.debug('Ignoring own ping (from %s)', addr[0])
continue
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = unserialize(msg[len('PING:'):])
scheduler_ip_addr = info['ip_addr']
scheduler_port = info['port']
assert info['version'] == _dispy_version
# scheduler_sign = info['sign']
assert isinstance(scheduler_port, int)
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
logger.debug(traceback.format_exc())
continue
logger.debug('relaying ping from %s / %s' % (info['ip_addr'], addr[0]))
if scheduler_ip_addr is None:
info['ip_addr'] = scheduler_ip_addr = addr[0]
relay_request = serialize(info)
bc_sock.sendto('PING:%s' % relay_request, ('<broadcast>', node_port))
else:
assert sock == sched_sock
msg, addr = sched_sock.recvfrom(1024)
if msg.startswith('PING:') and scheduler_ip_addr and scheduler_port:
try:
info = unserialize(msg[len('PONG:'):])
assert info['version'] == _dispy_version
assert isinstance(info['ip_addr'], str)
assert isinstance(info['port'], int)
# assert isinstance(info['cpus'], int)
info['scheduler_ip_addr'] = scheduler_ip_addr
relay_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
relay_sock.sendto('PING:' + serialize(info),
(scheduler_ip_addr, scheduler_port))
relay_sock.close()
except:
logger.debug(traceback.format_exc())
# raise
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
def __init__(self, cpus, ip_addr=None, ext_ip_addr=None, node_port=None,
scheduler_node=None, scheduler_port=None,
dest_path_prefix='', secret='', keyfile=None, certfile=None,
max_file_size=None, zombie_interval=60):
assert 0 < cpus <= multiprocessing.cpu_count()
self.cpus = cpus
if ip_addr:
ip_addr = _node_ipaddr(ip_addr)
if not ip_addr:
raise Exception('invalid ip_addr')
else:
self.name = socket.gethostname()
ip_addr = socket.gethostbyname(self.name)
if ext_ip_addr:
ext_ip_addr = _node_ipaddr(ext_ip_addr)
if not ext_ip_addr:
raise Exception('invalid ext_ip_addr')
else:
ext_ip_addr = ip_addr
try:
self.name = socket.gethostbyaddr(ext_ip_addr)[0]
except:
self.name = socket.gethostname()
if not node_port:
node_port = 51348
if not scheduler_port:
scheduler_port = 51347
self.ip_addr = ip_addr
self.ext_ip_addr = ext_ip_addr
self.scheduler_port = scheduler_port
self.pulse_interval = None
self.keyfile = keyfile
self.certfile = certfile
if self.keyfile:
self.keyfile = os.path.abspath(self.keyfile)
if self.certfile:
self.certfile = os.path.abspath(self.certfile)
self.asyncoro = AsynCoro()
self.tcp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.certfile:
self.tcp_sock = ssl.wrap_socket(self.tcp_sock, keyfile=self.keyfile,
certfile=self.certfile)
self.tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.tcp_sock.bind((self.ip_addr, node_port))
self.address = self.tcp_sock.getsockname()
self.tcp_sock.listen(30)
if dest_path_prefix:
self.dest_path_prefix = dest_path_prefix.strip().rstrip(os.sep)
else:
self.dest_path_prefix = os.path.join(os.sep, 'tmp', 'dispy')
if not os.path.isdir(self.dest_path_prefix):
os.makedirs(self.dest_path_prefix)
os.chmod(self.dest_path_prefix, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
if max_file_size is None:
max_file_size = MaxFileSize
self.max_file_size = max_file_size
self.avail_cpus = self.cpus
self.computations = {}
self.scheduler_ip_addr = None
self.file_uses = {}
self.job_infos = {}
self.lock = asyncoro.Lock()
self.terminate = False
self.signature = os.urandom(20).encode('hex')
self.auth_code = hashlib.sha1(self.signature + secret).hexdigest()
self.zombie_interval = 60 * zombie_interval
logger.debug('auth_code for %s: %s', ip_addr, self.auth_code)
self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.udp_sock.bind(('', node_port))
logger.info('serving %s cpus at %s:%s', self.cpus, self.ip_addr, node_port)
logger.debug('tcp server at %s:%s', self.address[0], self.address[1])
self.udp_sock = AsynCoroSocket(self.udp_sock, blocking=False)
scheduler_ip_addr = _node_ipaddr(scheduler_node)
self.reply_Q = multiprocessing.Queue()
self.reply_Q_thread = threading.Thread(target=self.__reply_Q)
self.reply_Q_thread.start()
self.timer_coro = Coro(self.timer_task)
# self.tcp_coro = Coro(self.tcp_server)
self.udp_coro = Coro(self.udp_server, scheduler_ip_addr)
def relay_pings(self,
ip_addr='',
netmask=None,
node_port=51348,
scheduler_node=None,
scheduler_port=51347):
netaddr = None
if not netmask:
try:
ip_addr, bits = ip_addr.split('/')
socket.inet_aton(ip_addr)
netmask = (0xffffffff << (32 - int(bits))) & 0xffffffff
netaddr = (struct.unpack(
'>L', socket.inet_aton(ip_addr))[0]) & netmask
except:
netmask = '255.255.255.255'
if ip_addr:
socket.inet_aton(ip_addr)
else:
ip_addr = socket.gethostbyname(socket.gethostname())
if not netaddr and netmask:
try:
if isinstance(netmask, str):
netmask = struct.unpack('>L', socket.inet_aton(netmask))[0]
else:
assert isinstance(netmask, int)
assert netmask > 0
netaddr = (struct.unpack(
'>L', socket.inet_aton(ip_addr))[0]) & netmask
except:
logger.warning('Invalid netmask')
try:
socket.inet_ntoa(struct.pack('>L', netaddr))
socket.inet_ntoa(struct.pack('>L', netmask))
except:
netaddr = netmask = None
bc_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
scheduler_ip_addrs = list(
filter(lambda ip: bool(ip), [_node_ipaddr(scheduler_node)]))
if scheduler_ip_addrs and scheduler_port:
relay_request = {
'ip_addrs': scheduler_ip_addrs,
'port': scheduler_port,
'version': _dispy_version,
'sign': None
}
bc_sock.sendto(b'PING:' + serialize(relay_request),
('<broadcast>', node_port))
bc_sock.close()
node_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
node_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
node_sock.bind(('', node_port))
sched_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sched_sock.bind(('', scheduler_port))
logger.info('Listening on %s:%s/%s', ip_addr, node_port,
scheduler_port)
while True:
ready = select.select([node_sock, sched_sock], [], [])[0]
for sock in ready:
if sock == node_sock:
msg, addr = node_sock.recvfrom(1024)
if not msg.startswith(b'PING:'):
logger.debug('Ignoring message "%s" from %s',
msg[:min(len(msg), 5)], addr[0])
continue
if netaddr and \
(struct.unpack('>L', socket.inet_aton(addr[0]))[0] & netmask) == netaddr:
logger.debug('Ignoring ping back (from %s)', addr[0])
continue
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = unserialize(msg[len(b'PING:'):])
if info['version'] != _dispy_version:
logger.warning(
'Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'],
_dispy_version)
continue
scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)',
addr[0], addr[1])
logger.debug(traceback.format_exc())
continue
logger.debug('relaying ping from %s / %s' %
(info['ip_addrs'], addr[0]))
bc_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST,
1)
bc_sock.sendto(b'PING:' + serialize(info),
('<broadcast>', node_port))
bc_sock.close()
else:
assert sock == sched_sock
msg, addr = sched_sock.recvfrom(1024)
if msg.startswith(
b'PING:'
) and scheduler_ip_addrs and scheduler_port:
try:
info = unserialize(msg[len(b'PING:'):])
if netaddr and info.get('scheduler_ip_addr', None) and \
(struct.unpack('>L', socket.inet_aton(info['scheduler_ip_addr']))[0] & netmask) == netaddr:
logger.debug('Ignoring ping back (from %s)' %
addr[0])
continue
assert info['version'] == _dispy_version
# assert isinstance(info['cpus'], int)
msg = {
'ip_addrs': scheduler_ip_addrs,
'port': scheduler_port,
'version': _dispy_version
}
relay_sock = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM)
relay_sock.sendto(b'PING:' + serialize(msg),
(info['ip_addr'], info['port']))
relay_sock.close()
except:
logger.debug(traceback.format_exc())
# raise
logger.debug('Ignoring ping message from %s (%s)',
addr[0], addr[1])
def __init__(self,
ip_addrs=[],
node_port=51348,
relay_port=0,
scheduler_nodes=[],
scheduler_port=51347,
ipv4_udp_multicast=False,
secret='',
certfile=None,
keyfile=None):
addrinfos = []
if not ip_addrs:
ip_addrs = [None]
for i in range(len(ip_addrs)):
ip_addr = ip_addrs[i]
addrinfo = dispy.host_addrinfo(host=ip_addr)
if not addrinfo:
logger.warning('Ignoring invalid ip_addr %s', ip_addr)
continue
addrinfos.append(addrinfo)
self.node_port = node_port
if not relay_port:
relay_port = node_port
self.relay_port = relay_port
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
self.ip_addrs = set()
self.scheduler_ip_addr = None
self.scheduler_port = scheduler_port
self.secret = secret
if certfile:
self.certfile = os.path.abspath(certfile)
else:
self.certfile = None
if keyfile:
self.keyfile = os.path.abspath(keyfile)
else:
self.keyfile = None
udp_addrinfos = {}
for addrinfo in addrinfos:
self.ip_addrs.add(addrinfo.ip)
if addrinfo.family == socket.AF_INET and self.ipv4_udp_multicast:
addrinfo.broadcast = dispy.IPV4_MULTICAST_GROUP
Task(self.relay_tcp_proc, addrinfo)
if os.name == 'nt':
bind_addr = addrinfo.ip
elif sys.platform == 'darwin':
if addrinfo.family == socket.AF_INET and (
not self.ipv4_udp_multicast):
bind_addr = ''
else:
bind_addr = addrinfo.broadcast
else:
bind_addr = addrinfo.broadcast
udp_addrinfos[bind_addr] = addrinfo
scheduler_ip_addrs = []
for addr in scheduler_nodes:
addr = dispy._node_ipaddr(addr)
if addr:
scheduler_ip_addrs.append(addr)
for bind_addr, addrinfo in udp_addrinfos.items():
Task(self.relay_udp_proc, bind_addr, addrinfo)
Task(self.sched_udp_proc, bind_addr, addrinfo)
for addr in scheduler_ip_addrs:
msg = {
'version': __version__,
'ip_addrs': [addr],
'port': self.scheduler_port,
'sign': None
}
Task(self.verify_broadcast, addrinfo, msg)
logger.info('version %s started', dispy._dispy_version)
def relay_pings(self, ip_addr='', netmask=None, node_port=51348,
scheduler_node=None, scheduler_port=51347):
netaddr = None
if not netmask:
try:
ip_addr, bits = ip_addr.split('/')
socket.inet_aton(ip_addr)
netmask = (0xffffffff << (32 - int(bits))) & 0xffffffff
netaddr = (struct.unpack('>L', socket.inet_aton(ip_addr))[0]) & netmask
except:
netmask = '255.255.255.255'
if ip_addr:
socket.inet_aton(ip_addr)
else:
ip_addr = socket.gethostbyname(socket.gethostname())
if not netaddr and netmask:
try:
if isinstance(netmask, str):
netmask = struct.unpack('>L', socket.inet_aton(netmask))[0]
else:
assert isinstance(netmask, int)
assert netmask > 0
netaddr = (struct.unpack('>L', socket.inet_aton(ip_addr))[0]) & netmask
except:
logger.warning('Invalid netmask')
try:
socket.inet_ntoa(struct.pack('>L', netaddr))
socket.inet_ntoa(struct.pack('>L', netmask))
except:
netaddr = netmask = None
scheduler_version = _dispy_version
bc_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bc_sock.bind(('', 0))
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
scheduler_ip_addr = _node_ipaddr(scheduler_node)
if scheduler_ip_addr and scheduler_port:
relay_request = serialize({'scheduler_ip_addr':scheduler_ip_addr,
'scheduler_port':scheduler_port,
'version':scheduler_version})
bc_sock.sendto('PING:%s' % relay_request, ('<broadcast>', node_port))
ping_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
ping_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ping_sock.bind(('', node_port))
pong_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
pong_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
pong_sock.bind(('', scheduler_port))
logger.info('Listening on %s:%s', ip_addr, node_port)
last_ping = 0
while True:
ready = select.select([ping_sock, pong_sock], [], [])[0]
for sock in ready:
if sock == ping_sock:
msg, addr = ping_sock.recvfrom(1024)
if not msg.startswith('PING:'):
logger.debug('Ignoring message "%s" from %s',
msg[:max(len(msg), 5)], addr[0])
continue
if netaddr and (struct.unpack('>L', socket.inet_aton(addr[0]))[0] & netmask) == netaddr:
logger.debug('Ignoring own ping (from %s)', addr[0])
continue
if (time.time() - last_ping) < 10:
logger.warning('Ignoring ping (from %s) for 10 more seconds', addr[0])
time.sleep(10)
last_ping = time.time()
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
data = unserialize(msg[len('PING:'):])
scheduler_ip_addr = data['scheduler_ip_addr']
scheduler_port = data['scheduler_port']
scheduler_version = data['version']
assert isinstance(scheduler_ip_addr, str)
assert isinstance(scheduler_port, int)
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0], addr[1])
continue
relay_request = serialize({'scheduler_ip_addr':scheduler_ip_addr,
'scheduler_port':scheduler_port,
'version':scheduler_version})
bc_sock.sendto('PING:%s' % relay_request, ('<broadcast>', node_port))
else:
assert sock == pong_sock
msg, addr = pong_sock.recvfrom(1024)
if not msg.startswith('PONG:'):
logger.debug('Ignoring pong message "%s" from %s',
msg[:max(len(msg), 5)], addr[0])
continue
# if netaddr and (struct.unpack('>L', socket.inet_aton(addr[0]))[0] & netmask) == netaddr:
# logger.debug('Ignoring own pong (from %s)', addr[0])
# continue
if not (scheduler_ip_addr and scheduler_port):
logger.debug('Ignoring pong message from %s', str(addr))
continue
logger.debug('Pong message from %s (%s)', addr[0], addr[1])
try:
pong = unserialize(msg[len('PONG:'):])
assert isinstance(pong['host'], str)
assert isinstance(pong['port'], int)
assert isinstance(pong['cpus'], int)
relay_request = serialize({'scheduler_ip_addr':scheduler_ip_addr,
'scheduler_port':scheduler_port,
'version':scheduler_version})
relay_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
relay_sock.sendto('PING:%s' % relay_request,
(pong['host'], node_port))
relay_sock.close()
except:
# raise
logger.debug('Ignoring pong message from %s (%s)', addr[0], addr[1])