def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2): pp.Server.__init__(self, ncpus, secret=secret, restart=restart, proto=proto) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = self.default_port self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Strarting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppcommon.start_thread("timeout_check", self.check_timeout)
def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None): pp.Server.__init__(self, ncpus, (), secret, restart, proto, socket_timeout) if pid_file: with open(pid_file, 'w') as pfile: six.print_(os.getpid(), file=pfile) atexit.register(os.remove, pid_file) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = ppc.randomport() self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Strarting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppc.start_thread("timeout_check", self.check_timeout)
def listen(self): """Listens for broadcasts from other clients/servers""" self.base.logger.debug("Listening (%s, %i)" % self.interface_addr) self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(5) self.socket.bind(self.interface_addr) ppcommon.start_thread("broadcast", self.broadcast) while True: try: if self.base._exiting: return message, (host, port) = self.socket.recvfrom(1024) remote_address = (host, self.broadcast_addr[1]) hostid = host + ":" + str(self.broadcast_addr[1]) self.base.logger.debug("Discovered host (%s, %i) message=%c" % (remote_address + (message[0], ))) if not self.base.autopp_list.get(hostid, 0) and self.isclient \ and message[0] == 'S': self.base.logger.debug("Connecting to host %s" % (hostid, )) ppcommon.start_thread("ppauto_connect1", self.base.connect1, remote_address+(False, )) if not self.isclient and message[0] == 'C': self.base.logger.debug("Replying to host %s" % (hostid, )) self.bsocket.sendto("S", self.broadcast_addr) except socket.timeout: pass except: self.base.logger.error("An error has occured during execution of " "Discover.listen") sys.excepthook(*sys.exc_info())
def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None): pp.Server.__init__(self, ncpus, (), secret, restart, proto, socket_timeout) if pid_file: with open(pid_file, 'w') as pfile: print >>pfile, os.getpid() atexit.register(os.remove, pid_file) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = self.default_port self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Strarting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppcommon.start_thread("timeout_check", self.check_timeout)
def __schedule_dedicated_workers(self): for job in self.__jobs.itervalues(): if job.worker.is_free and len(job.queue) > 0: job.worker.is_free = False self.__stats["local"].njobs += 1 self.__add_to_active_tasks(1) task = job.queue.pop() ppcommon.start_thread("run_local", self._run_local, task+(job.worker, job))
def __connect(self): """Connects to all remote ppservers""" for ppserver in self.ppservers: ppc.start_thread("connect1", self.connect1, ppserver) self.discover = ppauto.Discover(self, True) for ppserver in self.auto_ppservers: ppc.start_thread("discover.run", self.discover.run, ppserver)
def __connect(self): """Connects to all remote ppservers""" for ppserver in self.ppservers: ppcommon.start_thread("connect1", self.connect1, ppserver) self.discover = ppauto.Discover(self, True) for ppserver in self.auto_ppservers: ppcommon.start_thread("discover.run", self.discover.run, ppserver)
def listen(self): """Initiates listenting to incoming connections""" try: self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # following allows ppserver to restart faster on the same port self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT) self.ssocket.bind((self.host, self.port)) self.ssocket.listen(5) except socket.error: e = sys.exc_info()[1] self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e) try: while 1: csocket = None # accept connections from outside try: (csocket, address) = self.ssocket.accept() except socket.timeout: pass # don't exit on an interupt due to a signal except socket.error: e = sys.exc_info()[1] if e.errno == errno.EINTR: pass if self._exiting: return # now do something with the clientsocket # in this case, we'll pretend this is a threaded server if csocket: ppc.start_thread("client_socket", self.crun, (csocket, )) except KeyboardInterrupt: pass except: self.logger.debug("Exception in listen method (possibly expected)", exc_info=True) finally: self.logger.debug("Closing server socket") self.ssocket.close()
def __scheduler(self): """Schedules jobs for execution""" self.__queue_lock.acquire() while self.__queue: if self.__active_tasks < self.__ncpus: #TODO: select a job number on the basis of heuristic task = self.__queue.pop(0) for worker in self.__workers: if worker.is_free: worker.is_free = False break else: self.logger.error("There are no free workers left") raise RuntimeError("Error: No free workers") self.__add_to_active_tasks(1) try: self.__stats["local"].njobs += 1 ppc.start_thread("run_local", self._run_local, task + (worker, )) except: pass else: for rworker in self.__rworkers: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppc.start_thread("run_remote", self._run_remote, task + (rworker, )) break else: if len(self.__queue) > self.__ncpus: for rworker in self.__rworkers_reserved: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppc.start_thread("run_remote", self._run_remote, task + (rworker, )) break else: break else: break self.__queue_lock.release()
def __scheduler(self): """Schedules jobs for execution""" self.__queue_lock.acquire() while self.__queue: if self.__active_tasks < self.__ncpus: #TODO: select a job number on the basis of heuristic task = self.__queue.pop(0) for worker in self.__workers: if worker.is_free: worker.is_free = False break else: self.logger.error("There are no free workers left") raise RuntimeError("Error: No free workers") self.__add_to_active_tasks(1) try: self.__stats["local"].njobs += 1 ppcommon.start_thread("run_local", self._run_local, task+(worker, )) except: pass else: for rworker in self.__rworkers: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, )) break else: if len(self.__queue) > self.__ncpus: for rworker in self.__rworkers_reserved: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, )) break else: break else: break self.__queue_lock.release()
def __init__(self, ncpus="autodetect", ppservers=(), secret=None, restart=False, proto=2): """Creates Server instance ncpus - the number of worker processes to start on the local computer, if parameter is omitted it will be set to the number of processors in the system ppservers - list of active parallel python execution servers to connect with secret - passphrase for network connections, if omitted a default passphrase will be used. It's highly recommended to use a custom passphrase for all network connections. restart - whether to restart worker process after each task completion proto - protocol number for pickle module With ncpus = 1 all tasks are executed consequently For the best performance either use the default "autodetect" value or set ncpus to the total number of processors in the system """ if not isinstance(ppservers, tuple): raise TypeError("ppservers argument must be a tuple") self.logger = logging.getLogger('pp') self.logger.info("Creating server instance (pp-" + version+")") self.logger.info("Running on Python %s %s", sys.version.split(" ")[0], sys.platform) self.__tid = 0 self.__active_tasks = 0 self.__active_tasks_lock = threading.Lock() self.__queue = [] self.__queue_lock = threading.Lock() self.__jobs = {} self.__jobs_lock = threading.Lock() self.__workers = [] self.__dedicated_worker_pool = [] self.__dedicated_worker_pool_lock = threading.Lock() self.__rworkers = [] self.__rworkers_reserved = [] self.__sourcesHM = {} self.__sfuncHM = {} self.__waittasks = [] self.__waittasks_lock = threading.Lock() self._exiting = False self.__accurate_stats = True self.autopp_list = {} self.__active_rworkers_list_lock = threading.Lock() self.__restart_on_free = restart self.__pickle_proto = proto self.__remote_nodes = {} self.__remote_nodes_lock = threading.Lock() self.load_average = 0.0 self.__load_readings = collections.deque(maxlen=int(LOAD_AVERAGE_PERIOD / LOAD_AVERAGE_POLLING_INTERVAL)) self.__load_readings.extend([0] * self.__load_readings.maxlen) #initially the load is 0 self.__load_lock = threading.Lock() # add local directory and sys.path to PYTHONPATH pythondirs = [os.getcwd()] + sys.path if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]: pythondirs += os.environ["PYTHONPATH"].split(os.pathsep) os.environ["PYTHONPATH"] = os.pathsep.join(set(pythondirs)) atexit.register(self.destroy) self.__stats = {"local": _Statistics(0)} self.set_ncpus(ncpus) self.ppservers = [] self.auto_ppservers = [] for ppserver in ppservers: ppserver = ppserver.split(":") host = ppserver[0] if len(ppserver)>1: port = int(ppserver[1]) else: port = Server.default_port if host.find("*") == -1: self.ppservers.append((host, port)) else: if host == "*": host = "*.*.*.*" interface = host.replace("*", "0") broadcast = host.replace("*", "255") self.auto_ppservers.append(((interface, port), (broadcast, port))) self.__stats_lock = threading.Lock() ppcommon.start_thread("load", self._load_average_thread) if secret is not None: if not isinstance(secret, types.StringType): raise TypeError("secret must be of a string type") self.secret = str(secret) elif hasattr(user, "pp_secret"): secret = getattr(user, "pp_secret") if not isinstance(secret, types.StringType): raise TypeError("secret must be of a string type") self.secret = str(secret) else: self.secret = Server.default_secret self.__connect() self.__creation_time = time.time() print "pp local server started with %d workers, pid %d"% (self.__ncpus, os.getpid()) self.logger.info("pp local server started with %d workers" % (self.__ncpus, ))
def __send_local_task(self, task, worker, job): worker.is_free = False self.__add_to_active_tasks(1) self.__stats["local"].njobs += 1 ppcommon.start_thread("run_local", self._run_local, task+(worker, job))
def __send_remote_task(self, task, rworker, job): rworker.is_free = False self.__stats[rworker.id].njobs += 1 self.__add_to_active_tasks(1) ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, job))
def broadcast(self): """Initiaates auto-discovery mechanism""" discover = ppauto.Discover(self) ppc.start_thread("server_broadcast", discover.run, ((self.host, self.port), (self.bcast, self.port)))
def broadcast(self): """Initiaates auto-discovery mechanism""" discover = ppauto.Discover(self) ppcommon.start_thread("server_broadcast", discover.run, ((self.host, self.port), (self.bcast, self.port)))
class _NetworkServer(pp.Server): """Network Server Class """ def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None): pp.Server.__init__(self, ncpus, (), secret, restart, proto, socket_timeout) if pid_file: with open(pid_file, 'w') as pfile: print >>pfile, os.getpid() atexit.register(os.remove, pid_file) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = self.default_port self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Strarting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppcommon.start_thread("timeout_check", self.check_timeout) def ncon_add(self, val): """Keeps track of the number of connections and time of the last one""" self.ncon_lock.acquire() self.ncon += val self.last_con_time = time.time() self.ncon_lock.release() def check_timeout(self): """Checks if timeout happened and shutdowns server if it did""" while True: if self.ncon == 0: idle_time = time.time() - self.last_con_time if idle_time < self.timeout: time.sleep(self.timeout - idle_time) else: self.logger.debug("exiting ppserver due to timeout (no client"\ " connections in last %i sec)", self.timeout) os._exit(0) else: time.sleep(self.timeout) def listen(self): """Initiates listenting to incoming connections""" try: self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # following allows ppserver to restart faster on the same port self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT) self.ssocket.bind((self.host, self.port)) self.ssocket.listen(5) except socket.error, e: self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e) try: while 1: csocket = None # accept connections from outside try: (csocket, address) = self.ssocket.accept() except socket.timeout: pass # don't exit on an interupt due to a signal except socket.error as e: if e.errno == errno.EINTR: pass if self._exiting: return # now do something with the clientsocket # in this case, we'll pretend this is a threaded server if csocket: ppcommon.start_thread("client_socket", self.crun, (csocket, )) except KeyboardInterrupt: pass except: self.logger.debug("Exception in listen method (possibly expected)", exc_info=True) finally: self.logger.debug("Closing server socket") self.ssocket.close()