def __init__(self,
ncpus="autodetect",
interface="0.0.0.0",
broadcast="255.255.255.255",
port=None,
secret=None,
timeout=None,
restart=False,
proto=2):
pp.Server.__init__(self,
ncpus,
secret=secret,
restart=restart,
proto=proto)
self.host = interface
self.bcast = broadcast
if port is not None:
self.port = port
else:
self.port = self.default_port
self.timeout = timeout
self.ncon = 0
self.last_con_time = time.time()
self.ncon_lock = threading.Lock()
self.logger.debug("Strarting network server interface=%s port=%i" %
(self.host, self.port))
if self.timeout is not None:
self.logger.debug("ppserver will exit in %i seconds if no "\
"connections with clients exist" % (self.timeout))
ppcommon.start_thread("timeout_check", self.check_timeout)
def __init__(self, ncpus="autodetect", interface="0.0.0.0",
broadcast="255.255.255.255", port=None, secret=None,
timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None):
pp.Server.__init__(self, ncpus, (), secret, restart,
proto, socket_timeout)
if pid_file:
with open(pid_file, 'w') as pfile:
six.print_(os.getpid(), file=pfile)
atexit.register(os.remove, pid_file)
self.host = interface
self.bcast = broadcast
if port is not None:
self.port = port
else:
self.port = ppc.randomport()
self.timeout = timeout
self.ncon = 0
self.last_con_time = time.time()
self.ncon_lock = threading.Lock()
self.logger.debug("Strarting network server interface=%s port=%i"
% (self.host, self.port))
if self.timeout is not None:
self.logger.debug("ppserver will exit in %i seconds if no "\
"connections with clients exist" % (self.timeout))
ppc.start_thread("timeout_check", self.check_timeout)
def listen(self):
"""Listens for broadcasts from other clients/servers"""
self.base.logger.debug("Listening (%s, %i)" % self.interface_addr)
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.settimeout(5)
self.socket.bind(self.interface_addr)
ppcommon.start_thread("broadcast", self.broadcast)
while True:
try:
if self.base._exiting:
return
message, (host, port) = self.socket.recvfrom(1024)
remote_address = (host, self.broadcast_addr[1])
hostid = host + ":" + str(self.broadcast_addr[1])
self.base.logger.debug("Discovered host (%s, %i) message=%c"
% (remote_address + (message[0], )))
if not self.base.autopp_list.get(hostid, 0) and self.isclient \
and message[0] == 'S':
self.base.logger.debug("Connecting to host %s" % (hostid, ))
ppcommon.start_thread("ppauto_connect1", self.base.connect1,
remote_address+(False, ))
if not self.isclient and message[0] == 'C':
self.base.logger.debug("Replying to host %s" % (hostid, ))
self.bsocket.sendto("S", self.broadcast_addr)
except socket.timeout:
pass
except:
self.base.logger.error("An error has occured during execution of "
"Discover.listen")
sys.excepthook(*sys.exc_info())
def __init__(self, ncpus="autodetect", interface="0.0.0.0",
broadcast="255.255.255.255", port=None, secret=None,
timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None):
pp.Server.__init__(self, ncpus, (), secret, restart,
proto, socket_timeout)
if pid_file:
with open(pid_file, 'w') as pfile:
print >>pfile, os.getpid()
atexit.register(os.remove, pid_file)
self.host = interface
self.bcast = broadcast
if port is not None:
self.port = port
else:
self.port = self.default_port
self.timeout = timeout
self.ncon = 0
self.last_con_time = time.time()
self.ncon_lock = threading.Lock()
self.logger.debug("Strarting network server interface=%s port=%i"
% (self.host, self.port))
if self.timeout is not None:
self.logger.debug("ppserver will exit in %i seconds if no "\
"connections with clients exist" % (self.timeout))
ppcommon.start_thread("timeout_check", self.check_timeout)
def __schedule_dedicated_workers(self):
for job in self.__jobs.itervalues():
if job.worker.is_free and len(job.queue) > 0:
job.worker.is_free = False
self.__stats["local"].njobs += 1
self.__add_to_active_tasks(1)
task = job.queue.pop()
ppcommon.start_thread("run_local", self._run_local, task+(job.worker, job))
def __connect(self):
"""Connects to all remote ppservers"""
for ppserver in self.ppservers:
ppc.start_thread("connect1", self.connect1, ppserver)
self.discover = ppauto.Discover(self, True)
for ppserver in self.auto_ppservers:
ppc.start_thread("discover.run", self.discover.run, ppserver)
def __connect(self):
"""Connects to all remote ppservers"""
for ppserver in self.ppservers:
ppcommon.start_thread("connect1", self.connect1, ppserver)
self.discover = ppauto.Discover(self, True)
for ppserver in self.auto_ppservers:
ppcommon.start_thread("discover.run", self.discover.run, ppserver)
def listen(self):
"""Initiates listenting to incoming connections"""
try:
self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# following allows ppserver to restart faster on the same port
self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT)
self.ssocket.bind((self.host, self.port))
self.ssocket.listen(5)
except socket.error:
e = sys.exc_info()[1]
self.logger.error("Cannot create socket for %s:%s, %s", self.host,
self.port, e)
try:
while 1:
csocket = None
# accept connections from outside
try:
(csocket, address) = self.ssocket.accept()
except socket.timeout:
pass
# don't exit on an interupt due to a signal
except socket.error:
e = sys.exc_info()[1]
if e.errno == errno.EINTR:
pass
if self._exiting:
return
# now do something with the clientsocket
# in this case, we'll pretend this is a threaded server
if csocket:
ppc.start_thread("client_socket", self.crun, (csocket, ))
except KeyboardInterrupt:
pass
except:
self.logger.debug("Exception in listen method (possibly expected)",
exc_info=True)
finally:
self.logger.debug("Closing server socket")
self.ssocket.close()
def listen(self):
"""Initiates listenting to incoming connections"""
try:
self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# following allows ppserver to restart faster on the same port
self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT)
self.ssocket.bind((self.host, self.port))
self.ssocket.listen(5)
except socket.error:
e = sys.exc_info()[1]
self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e)
try:
while 1:
csocket = None
# accept connections from outside
try:
(csocket, address) = self.ssocket.accept()
except socket.timeout:
pass
# don't exit on an interupt due to a signal
except socket.error:
e = sys.exc_info()[1]
if e.errno == errno.EINTR:
pass
if self._exiting:
return
# now do something with the clientsocket
# in this case, we'll pretend this is a threaded server
if csocket:
ppc.start_thread("client_socket", self.crun, (csocket, ))
except KeyboardInterrupt:
pass
except:
self.logger.debug("Exception in listen method (possibly expected)", exc_info=True)
finally:
self.logger.debug("Closing server socket")
self.ssocket.close()
def __init__(self, ncpus="autodetect", interface="0.0.0.0",
broadcast="255.255.255.255", port=None, secret=None,
timeout=None, restart=False, proto=2):
pp.Server.__init__(self, ncpus, secret=secret, restart=restart,
proto=proto)
self.host = interface
self.bcast = broadcast
if port is not None:
self.port = port
else:
self.port = self.default_port
self.timeout = timeout
self.ncon = 0
self.last_con_time = time.time()
self.ncon_lock = threading.Lock()
self.logger.debug("Strarting network server interface=%s port=%i"
% (self.host, self.port))
if self.timeout is not None:
self.logger.debug("ppserver will exit in %i seconds if no "\
"connections with clients exist" % (self.timeout))
ppcommon.start_thread("timeout_check", self.check_timeout)
def __scheduler(self):
"""Schedules jobs for execution"""
self.__queue_lock.acquire()
while self.__queue:
if self.__active_tasks < self.__ncpus:
#TODO: select a job number on the basis of heuristic
task = self.__queue.pop(0)
for worker in self.__workers:
if worker.is_free:
worker.is_free = False
break
else:
self.logger.error("There are no free workers left")
raise RuntimeError("Error: No free workers")
self.__add_to_active_tasks(1)
try:
self.__stats["local"].njobs += 1
ppc.start_thread("run_local", self._run_local,
task + (worker, ))
except:
pass
else:
for rworker in self.__rworkers:
if rworker.is_free:
rworker.is_free = False
task = self.__queue.pop(0)
self.__stats[rworker.id].njobs += 1
ppc.start_thread("run_remote", self._run_remote,
task + (rworker, ))
break
else:
if len(self.__queue) > self.__ncpus:
for rworker in self.__rworkers_reserved:
if rworker.is_free:
rworker.is_free = False
task = self.__queue.pop(0)
self.__stats[rworker.id].njobs += 1
ppc.start_thread("run_remote",
self._run_remote,
task + (rworker, ))
break
else:
break
else:
break
self.__queue_lock.release()
def __scheduler(self):
"""Schedules jobs for execution"""
self.__queue_lock.acquire()
while self.__queue:
if self.__active_tasks < self.__ncpus:
#TODO: select a job number on the basis of heuristic
task = self.__queue.pop(0)
for worker in self.__workers:
if worker.is_free:
worker.is_free = False
break
else:
self.logger.error("There are no free workers left")
raise RuntimeError("Error: No free workers")
self.__add_to_active_tasks(1)
try:
self.__stats["local"].njobs += 1
ppcommon.start_thread("run_local", self._run_local, task+(worker, ))
except:
pass
else:
for rworker in self.__rworkers:
if rworker.is_free:
rworker.is_free = False
task = self.__queue.pop(0)
self.__stats[rworker.id].njobs += 1
ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, ))
break
else:
if len(self.__queue) > self.__ncpus:
for rworker in self.__rworkers_reserved:
if rworker.is_free:
rworker.is_free = False
task = self.__queue.pop(0)
self.__stats[rworker.id].njobs += 1
ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, ))
break
else:
break
else:
break
self.__queue_lock.release()
def __init__(self, ncpus="autodetect", ppservers=(), secret=None,
restart=False, proto=2):
"""Creates Server instance
ncpus - the number of worker processes to start on the local
computer, if parameter is omitted it will be set to
the number of processors in the system
ppservers - list of active parallel python execution servers
to connect with
secret - passphrase for network connections, if omitted a default
passphrase will be used. It's highly recommended to use a
custom passphrase for all network connections.
restart - whether to restart worker process after each task completion
proto - protocol number for pickle module
With ncpus = 1 all tasks are executed consequently
For the best performance either use the default "autodetect" value
or set ncpus to the total number of processors in the system
"""
if not isinstance(ppservers, tuple):
raise TypeError("ppservers argument must be a tuple")
self.logger = logging.getLogger('pp')
self.logger.info("Creating server instance (pp-" + version+")")
self.logger.info("Running on Python %s %s", sys.version.split(" ")[0],
sys.platform)
self.__tid = 0
self.__active_tasks = 0
self.__active_tasks_lock = threading.Lock()
self.__queue = []
self.__queue_lock = threading.Lock()
self.__jobs = {}
self.__jobs_lock = threading.Lock()
self.__workers = []
self.__dedicated_worker_pool = []
self.__dedicated_worker_pool_lock = threading.Lock()
self.__rworkers = []
self.__rworkers_reserved = []
self.__sourcesHM = {}
self.__sfuncHM = {}
self.__waittasks = []
self.__waittasks_lock = threading.Lock()
self._exiting = False
self.__accurate_stats = True
self.autopp_list = {}
self.__active_rworkers_list_lock = threading.Lock()
self.__restart_on_free = restart
self.__pickle_proto = proto
self.__remote_nodes = {}
self.__remote_nodes_lock = threading.Lock()
self.load_average = 0.0
self.__load_readings = collections.deque(maxlen=int(LOAD_AVERAGE_PERIOD / LOAD_AVERAGE_POLLING_INTERVAL))
self.__load_readings.extend([0] * self.__load_readings.maxlen) #initially the load is 0
self.__load_lock = threading.Lock()
# add local directory and sys.path to PYTHONPATH
pythondirs = [os.getcwd()] + sys.path
if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]:
pythondirs += os.environ["PYTHONPATH"].split(os.pathsep)
os.environ["PYTHONPATH"] = os.pathsep.join(set(pythondirs))
atexit.register(self.destroy)
self.__stats = {"local": _Statistics(0)}
self.set_ncpus(ncpus)
self.ppservers = []
self.auto_ppservers = []
for ppserver in ppservers:
ppserver = ppserver.split(":")
host = ppserver[0]
if len(ppserver)>1:
port = int(ppserver[1])
else:
port = Server.default_port
if host.find("*") == -1:
self.ppservers.append((host, port))
else:
if host == "*":
host = "*.*.*.*"
interface = host.replace("*", "0")
broadcast = host.replace("*", "255")
self.auto_ppservers.append(((interface, port),
(broadcast, port)))
self.__stats_lock = threading.Lock()
ppcommon.start_thread("load", self._load_average_thread)
if secret is not None:
if not isinstance(secret, types.StringType):
raise TypeError("secret must be of a string type")
self.secret = str(secret)
elif hasattr(user, "pp_secret"):
secret = getattr(user, "pp_secret")
if not isinstance(secret, types.StringType):
raise TypeError("secret must be of a string type")
self.secret = str(secret)
else:
self.secret = Server.default_secret
self.__connect()
self.__creation_time = time.time()
print "pp local server started with %d workers, pid %d"% (self.__ncpus, os.getpid())
self.logger.info("pp local server started with %d workers"
% (self.__ncpus, ))
def __send_local_task(self, task, worker, job):
worker.is_free = False
self.__add_to_active_tasks(1)
self.__stats["local"].njobs += 1
ppcommon.start_thread("run_local", self._run_local, task+(worker, job))
def __send_remote_task(self, task, rworker, job):
rworker.is_free = False
self.__stats[rworker.id].njobs += 1
self.__add_to_active_tasks(1)
ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, job))
def broadcast(self):
"""Initiaates auto-discovery mechanism"""
discover = ppauto.Discover(self)
ppc.start_thread("server_broadcast", discover.run,
((self.host, self.port), (self.bcast, self.port)))
def broadcast(self):
"""Initiaates auto-discovery mechanism"""
discover = ppauto.Discover(self)
ppcommon.start_thread("server_broadcast", discover.run,
((self.host, self.port), (self.bcast, self.port)))
class _NetworkServer(pp.Server):
"""Network Server Class
"""
def __init__(self, ncpus="autodetect", interface="0.0.0.0",
broadcast="255.255.255.255", port=None, secret=None,
timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None):
pp.Server.__init__(self, ncpus, (), secret, restart,
proto, socket_timeout)
if pid_file:
with open(pid_file, 'w') as pfile:
print >>pfile, os.getpid()
atexit.register(os.remove, pid_file)
self.host = interface
self.bcast = broadcast
if port is not None:
self.port = port
else:
self.port = self.default_port
self.timeout = timeout
self.ncon = 0
self.last_con_time = time.time()
self.ncon_lock = threading.Lock()
self.logger.debug("Strarting network server interface=%s port=%i"
% (self.host, self.port))
if self.timeout is not None:
self.logger.debug("ppserver will exit in %i seconds if no "\
"connections with clients exist" % (self.timeout))
ppcommon.start_thread("timeout_check", self.check_timeout)
def ncon_add(self, val):
"""Keeps track of the number of connections and time of the last one"""
self.ncon_lock.acquire()
self.ncon += val
self.last_con_time = time.time()
self.ncon_lock.release()
def check_timeout(self):
"""Checks if timeout happened and shutdowns server if it did"""
while True:
if self.ncon == 0:
idle_time = time.time() - self.last_con_time
if idle_time < self.timeout:
time.sleep(self.timeout - idle_time)
else:
self.logger.debug("exiting ppserver due to timeout (no client"\
" connections in last %i sec)", self.timeout)
os._exit(0)
else:
time.sleep(self.timeout)
def listen(self):
"""Initiates listenting to incoming connections"""
try:
self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# following allows ppserver to restart faster on the same port
self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT)
self.ssocket.bind((self.host, self.port))
self.ssocket.listen(5)
except socket.error, e:
self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e)
try:
while 1:
csocket = None
# accept connections from outside
try:
(csocket, address) = self.ssocket.accept()
except socket.timeout:
pass
# don't exit on an interupt due to a signal
except socket.error as e:
if e.errno == errno.EINTR:
pass
if self._exiting:
return
# now do something with the clientsocket
# in this case, we'll pretend this is a threaded server
if csocket:
ppcommon.start_thread("client_socket", self.crun, (csocket, ))
except KeyboardInterrupt:
pass
except:
self.logger.debug("Exception in listen method (possibly expected)", exc_info=True)
finally:
self.logger.debug("Closing server socket")
self.ssocket.close()
