def __init__(self, ncpus="autodetect", interface="0.0.0.0",
broadcast="255.255.255.255", port=None, secret=None,
timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None):
pp.Server.__init__(self, ncpus, (), secret, restart,
proto, socket_timeout)
if pid_file:
with open(pid_file, 'w') as pfile:
six.print_(os.getpid(), file=pfile)
atexit.register(os.remove, pid_file)
self.host = interface
self.bcast = broadcast
if port is not None:
self.port = port
else:
self.port = ppc.randomport()
self.timeout = timeout
self.ncon = 0
self.last_con_time = time.time()
self.ncon_lock = threading.Lock()
self.logger.debug("Strarting network server interface=%s port=%i"
% (self.host, self.port))
if self.timeout is not None:
self.logger.debug("ppserver will exit in %i seconds if no "\
"connections with clients exist" % (self.timeout))
ppc.start_thread("timeout_check", self.check_timeout)
def __init__(self,
ncpus="autodetect",
interface="0.0.0.0",
broadcast="255.255.255.255",
port=None,
secret=None,
timeout=None,
restart=False,
proto=2,
socket_timeout=3600,
pid_file=None):
pp.Server.__init__(self, ncpus, (), secret, restart, proto,
socket_timeout)
if pid_file:
with open(pid_file, 'w') as pfile:
six.print_(os.getpid(), file=pfile)
atexit.register(os.remove, pid_file)
self.host = interface
self.bcast = broadcast
if port is not None:
self.port = port
else:
self.port = ppc.randomport()
self.timeout = timeout
self.ncon = 0
self.last_con_time = time.time()
self.ncon_lock = threading.Lock()
self.logger.debug("Strarting network server interface=%s port=%i" %
(self.host, self.port))
if self.timeout is not None:
self.logger.debug("ppserver will exit in %i seconds if no "\
"connections with clients exist" % (self.timeout))
ppc.start_thread("timeout_check", self.check_timeout)
def __init__(self,
ncpus="autodetect",
ppservers=(),
secret=None,
restart=False,
proto=2,
socket_timeout=3600):
"""Creates Server instance
ncpus - the number of worker processes to start on the local
computer, if parameter is omitted it will be set to
the number of processors in the system
ppservers - list of active parallel python execution servers
to connect with
secret - passphrase for network connections, if omitted a default
passphrase will be used. It's highly recommended to use a
custom passphrase for all network connections.
restart - whether to restart worker process after each task completion
proto - protocol number for pickle module
socket_timeout - socket timeout in seconds which is also the maximum
time a remote job could be executed. Increase this value
if you have long running jobs or decrease if connectivity
to remote ppservers is often lost.
With ncpus = 1 all tasks are executed consequently
For the best performance either use the default "autodetect" value
or set ncpus to the total number of processors in the system
"""
if not isinstance(ppservers, tuple):
raise TypeError("ppservers argument must be a tuple")
self.logger = logging.getLogger('pp')
self.logger.info("Creating server instance (pp-" + version + ")")
self.logger.info("Running on Python %s %s",
sys.version.split(" ")[0], sys.platform)
self.__tid = 0
self.__active_tasks = 0
self.__active_tasks_lock = threading.Lock()
self.__queue = []
self.__queue_lock = threading.Lock()
self.__workers = []
self.__rworkers = []
self.__rworkers_reserved = []
self.__sourcesHM = {}
self.__sfuncHM = {}
self.__waittasks = []
self.__waittasks_lock = threading.Lock()
self._exiting = False
self.__accurate_stats = True
self.autopp_list = {}
self.__active_rworkers_list_lock = threading.Lock()
self.__restart_on_free = restart
self.__pickle_proto = proto
self.__connect_locks = {}
# add local directory and sys.path to PYTHONPATH
pythondirs = [os.getcwd()] + sys.path
if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]:
pythondirs += os.environ["PYTHONPATH"].split(os.pathsep)
os.environ["PYTHONPATH"] = os.pathsep.join(set(pythondirs))
atexit.register(self.destroy)
self.__stats = {"local": _Statistics(0)}
self.set_ncpus(ncpus)
self.ppservers = []
self.auto_ppservers = []
self.socket_timeout = socket_timeout
for ppserver in ppservers:
ppserver = ppserver.split(":")
host = ppserver[0]
if len(ppserver) > 1:
port = int(ppserver[1])
else:
port = ppc.randomport()
if host.find("*") == -1:
self.ppservers.append((host, port))
else:
if host == "*":
host = "*.*.*.*"
interface = host.replace("*", "0")
broadcast = host.replace("*", "255")
self.auto_ppservers.append(
((interface, port), (broadcast, port)))
self.__stats_lock = threading.Lock()
if secret is not None:
if not isinstance(secret, str):
raise TypeError("secret must be of a string type")
self.secret = str(secret)
elif hasattr(user, "pp_secret"):
secret = getattr(user, "pp_secret")
if not isinstance(secret, str):
raise TypeError("secret must be of a string type")
self.secret = str(secret)
else:
self.secret = Server.default_secret
self.__connect()
self.__creation_time = time.time()
self.logger.info("pp local server started with %d workers" %
(self.__ncpus, ))
def __init__(self, ncpus="autodetect", ppservers=(), secret=None,
restart=False, proto=2, socket_timeout=3600):
"""Creates Server instance
ncpus - the number of worker processes to start on the local
computer, if parameter is omitted it will be set to
the number of processors in the system
ppservers - list of active parallel python execution servers
to connect with
secret - passphrase for network connections, if omitted a default
passphrase will be used. It's highly recommended to use a
custom passphrase for all network connections.
restart - whether to restart worker process after each task completion
proto - protocol number for pickle module
socket_timeout - socket timeout in seconds which is also the maximum
time a remote job could be executed. Increase this value
if you have long running jobs or decrease if connectivity
to remote ppservers is often lost.
With ncpus = 1 all tasks are executed consequently
For the best performance either use the default "autodetect" value
or set ncpus to the total number of processors in the system
"""
if not isinstance(ppservers, tuple):
raise TypeError("ppservers argument must be a tuple")
self.logger = logging.getLogger('pp')
self.logger.info("Creating server instance (pp-" + version+")")
self.logger.info("Running on Python %s %s", sys.version.split(" ")[0],
sys.platform)
self.__tid = 0
self.__active_tasks = 0
self.__active_tasks_lock = threading.Lock()
self.__queue = []
self.__queue_lock = threading.Lock()
self.__workers = []
self.__rworkers = []
self.__rworkers_reserved = []
self.__sourcesHM = {}
self.__sfuncHM = {}
self.__waittasks = []
self.__waittasks_lock = threading.Lock()
self._exiting = False
self.__accurate_stats = True
self.autopp_list = {}
self.__active_rworkers_list_lock = threading.Lock()
self.__restart_on_free = restart
self.__pickle_proto = proto
self.__connect_locks = {}
# add local directory and sys.path to PYTHONPATH
pythondirs = [os.getcwd()] + sys.path
if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]:
pythondirs += os.environ["PYTHONPATH"].split(os.pathsep)
os.environ["PYTHONPATH"] = os.pathsep.join(set(pythondirs))
atexit.register(self.destroy)
self.__stats = {"local": _Statistics(0)}
self.set_ncpus(ncpus)
self.ppservers = []
self.auto_ppservers = []
self.socket_timeout = socket_timeout
for ppserver in ppservers:
ppserver = ppserver.split(":")
host = ppserver[0]
if len(ppserver)>1:
port = int(ppserver[1])
else:
port = ppc.randomport()
if host.find("*") == -1:
self.ppservers.append((host, port))
else:
if host == "*":
host = "*.*.*.*"
interface = host.replace("*", "0")
broadcast = host.replace("*", "255")
self.auto_ppservers.append(((interface, port),
(broadcast, port)))
self.__stats_lock = threading.Lock()
if secret is not None:
if not isinstance(secret, str):
raise TypeError("secret must be of a string type")
self.secret = str(secret)
elif hasattr(user, "pp_secret"):
secret = getattr(user, "pp_secret")
if not isinstance(secret, str):
raise TypeError("secret must be of a string type")
self.secret = str(secret)
else:
self.secret = Server.default_secret
self.__connect()
self.__creation_time = time.time()
self.logger.info("pp local server started with %d workers"
% (self.__ncpus, ))
