def tcp_server(self, addrinfo, task=None):
task.set_daemon()
sock = AsyncSocket(socket.socket(addrinfo.family, socket.SOCK_STREAM),
keyfile=self.keyfile,
certfile=self.certfile)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((addrinfo.ip, self.info_port))
except Exception:
logger.warning('Could not bind TCP server to %s:%s', addrinfo.ip,
self.info_port)
raise StopIteration
logger.info('dispyadmin TCP server at %s:%s', addrinfo.ip,
self.info_port)
sock.listen(16)
while 1:
try:
conn, addr = yield sock.accept()
except ssl.SSLError as err:
logger.debug('SSL connection failed: %s', str(err))
continue
except GeneratorExit:
break
except Exception:
logger.debug(traceback.format_exc())
continue
Task(self.tcp_req, conn, addr)
sock.close()
def __init__(self, cluster, host='', port=8181, poll_sec=10, DocumentRoot=None,
keyfile=None, certfile=None):
self._cluster_lock = threading.Lock()
self._clusters = {}
if cluster:
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
if cluster.status_callback is None:
cluster.status_callback = functools.partial(self.cluster_status, cluster_info)
if not DocumentRoot:
DocumentRoot = os.path.join(os.path.dirname(__file__), 'data')
if poll_sec < 1:
logger.warning('invalid poll_sec value %s; it must be at least 1' % poll_sec)
poll_sec = 1
self._poll_sec = poll_sec
self._http_handler = None
self._server = HTTPServer((host, port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s' %
('s' if certfile else '', str(self._server.socket.getsockname())))
def __init__(self, DocumentRoot, secret='', http_host='localhost', http_port=8181,
info_port=51347, node_port=51348, poll_interval=60, ping_interval=600,
ip_addrs=[], ipv4_udp_multicast=False, certfile=None, keyfile=None):
self.lock = threading.Lock()
self.client_uid = None
self.client_uid_time = 0
self.nodes = {}
self.updates = {}
if poll_interval < 1:
logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval)
poll_interval = 1
self.info_port = info_port
self.poll_interval = poll_interval
self.ping_interval = ping_interval
self.secret = secret
self.node_port = node_port
self.keyfile = keyfile
self.certfile = certfile
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
self.addrinfos = {}
if not ip_addrs:
ip_addrs = [None]
for i in range(len(ip_addrs)):
ip_addr = ip_addrs[i]
addrinfo = dispy.host_addrinfo(host=ip_addr, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid ip_addr %s', ip_addr)
continue
self.addrinfos[addrinfo.ip] = addrinfo
if not self.addrinfos:
raise Exception('No valid IP address found')
self.http_port = http_port
self.sign = hashlib.sha1(os.urandom(20))
for ip_addr in self.addrinfos:
self.sign.update(ip_addr.encode())
self.sign = self.sign.hexdigest()
self.auth = dispy.auth_code(self.secret, self.sign)
self.tcp_tasks = []
self.udp_tasks = []
udp_addrinfos = {}
for addrinfo in self.addrinfos.values():
self.tcp_tasks.append(Task(self.tcp_server, addrinfo))
udp_addrinfos[addrinfo.bind_addr] = addrinfo
for bind_addr, addrinfo in udp_addrinfos.items():
self.udp_tasks.append(Task(self.udp_server, addrinfo))
self._server = HTTPServer((http_host, http_port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self.timer = Task(self.timer_proc)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s', 's' if certfile else '',
':'.join(map(str, self._server.socket.getsockname())))
def __init__(self, cluster, host='', port=8181, poll_sec=10, DocumentRoot=None,
keyfile=None, certfile=None):
self._cluster_lock = threading.Lock()
self._clusters = {}
if cluster:
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
if cluster.status_callback is None:
cluster.status_callback = functools.partial(self.cluster_status, cluster_info)
if not DocumentRoot:
DocumentRoot = os.path.join(os.path.dirname(__file__), 'data')
if poll_sec < 1:
logger.warning('invalid poll_sec value %s; it must be at least 1' % poll_sec)
poll_sec = 1
self._poll_sec = poll_sec
self._http_handler = None
self._server = HTTPServer((host, port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s' %
('s' if certfile else '', str(self._server.socket.getsockname())))
def __init__(self, DocumentRoot, secret='', http_host='localhost',
poll_interval=60, ping_interval=600, hosts=[], ipv4_udp_multicast=False,
certfile=None, keyfile=None):
http_port = dispy.config.HTTPServerPort
self.node_port = eval(dispy.config.NodePort)
self.info_port = eval(dispy.config.ClientPort)
self.lock = threading.Lock()
self.client_uid = None
self.client_uid_time = 0
self.nodes = {}
self.updates = {}
if poll_interval < 1:
logger.warning('invalid poll_interval value %s; it must be at least 1', poll_interval)
poll_interval = 1
self.poll_interval = poll_interval
self.ping_interval = ping_interval
self.secret = secret
self.keyfile = keyfile
self.certfile = certfile
self.ipv4_udp_multicast = bool(ipv4_udp_multicast)
self.addrinfos = []
if not hosts:
hosts = [None]
for host in hosts:
addrinfo = dispy.host_addrinfo(host=host, ipv4_multicast=self.ipv4_udp_multicast)
if not addrinfo:
logger.warning('Ignoring invalid host %s', host)
continue
self.addrinfos.append(addrinfo)
if not self.addrinfos:
raise Exception('No valid host name / IP address found')
self.sign = hashlib.sha1(os.urandom(20))
for addrinfo in self.addrinfos:
self.sign.update(addrinfo.ip.encode())
self.sign = self.sign.hexdigest()
self.auth = dispy.auth_code(self.secret, self.sign)
self.tcp_tasks = []
self.udp_tasks = []
udp_addrinfos = {}
for addrinfo in self.addrinfos:
self.tcp_tasks.append(Task(self.tcp_server, addrinfo))
udp_addrinfos[addrinfo.bind_addr] = addrinfo
for bind_addr, addrinfo in udp_addrinfos.items():
self.udp_tasks.append(Task(self.udp_server, addrinfo))
self._server = HTTPServer((http_host, http_port), lambda *args:
self.__class__._HTTPRequestHandler(self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket, keyfile=keyfile,
certfile=certfile, server_side=True)
self.timer = Task(self.timer_proc)
self._httpd_thread = threading.Thread(target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
self.client_host = self._server.socket.getsockname()[0]
logger.info('Started HTTP%s server at %s:%s', 's' if certfile else '',
self.client_host, self._server.socket.getsockname()[1])
def shutdown(self, wait=True):
"""This method should be called by user program to close the
http server.
"""
if wait:
logger.info(
'HTTP server waiting for %s seconds for client updates before quitting'
% self.poll_interval)
time.sleep(self.poll_interval)
self._server.shutdown()
self._server.server_close()
def shutdown(self, wait=True):
"""This method should be called by user program to close the
http server.
"""
if wait:
logger.info(
'HTTP server waiting for %s seconds for client updates before quitting',
self._poll_sec)
time.sleep(self._poll_sec)
self._server.shutdown()
self._server.server_close()
def __init__(self,
cluster,
host='',
port=int(dispy.config.HTTPServerPort),
poll_sec=10,
DocumentRoot=None,
keyfile=None,
certfile=None,
show_job_args=True):
self._cluster_lock = threading.Lock()
self._clusters = {}
if cluster:
cluster_info = self.__class__._ClusterInfo(cluster)
self._clusters[cluster.name] = cluster_info
http_callback = functools.partial(self.cluster_status,
cluster_info)
if cluster.status_callback:
client_callback = cluster.status_callback
def chain_callbacks(status, node, job):
http_callback(status, node, job)
client_callback(status, node, job)
cluster.status_callback = chain_callbacks
else:
cluster.status_callback = http_callback
if not DocumentRoot:
DocumentRoot = os.path.join(os.path.dirname(__file__), 'data')
if poll_sec < 1:
logger.warning('invalid poll_sec value %s; it must be at least 1',
poll_sec)
poll_sec = 1
self._poll_sec = poll_sec
self._show_args = bool(show_job_args)
self._server = HTTPServer(
(host, int(port)),
lambda *args: self.__class__._HTTPRequestHandler(
self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket,
keyfile=keyfile,
certfile=certfile,
server_side=True)
self._httpd_thread = threading.Thread(
target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
logger.info('Started HTTP%s server at %s', 's' if certfile else '',
str(self._server.socket.getsockname()))
else:
logger.setLevel(logger.INFO)
del config['loglevel']
if config['certfile']:
config['certfile'] = os.path.abspath(config['certfile'])
else:
config['certfile'] = None
if config['keyfile']:
config['keyfile'] = os.path.abspath(config['keyfile'])
else:
config['keyfile'] = None
args = config.pop('args')
logger.info('dispy version %s', _dispy_version)
if config['scheduler_node']:
cluster = SharedJobCluster(**config)
else:
del config['scheduler_node']
cluster = JobCluster(**config)
jobs = []
for n, arg in enumerate(args, start=1):
job = cluster.submit(*(arg.split()))
job.id = n
jobs.append((job, arg))
for job, args in jobs:
job()