def __init__(self,
group=":PyUtilibServer",
host=None,
port=None,
num_dispatcher_tries=30,
caller_name="Task Worker",
verbose=False,
name=None):
self._verbose = verbose
# A worker can set this flag
# if an error occurs during processing
self._worker_error = False
self._worker_shutdown = False
self._worker_task_return_queue = _worker_task_return_queue_unset
# sets whether or not multiple tasks should
# be gathered from the worker queue during
# each request for work
self._bulk_task_collection = False
if _pyro is None:
raise ImportError("Pyro or Pyro4 is not available")
# Deprecated in Pyro3
# Removed in Pyro4
if using_pyro3:
_pyro.core.initClient()
if name is None:
self.WORKERNAME = "Worker_%d@%s" % (os.getpid(),
socket.gethostname())
else:
self.WORKERNAME = name
self.ns = get_nameserver(host=host, port=port, caller_name=caller_name)
if self.ns is None:
raise RuntimeError("TaskWorkerBase failed to locate "
"Pyro name server on the network!")
print('Worker attempting to find Pyro dispatcher object...')
cumulative_sleep_time = 0.0
self.dispatcher = None
for i in xrange(0, num_dispatcher_tries):
dispatchers = get_dispatchers(group=group, ns=self.ns)
random.shuffle(dispatchers)
for name, uri in dispatchers:
try:
if using_pyro3:
self.dispatcher = _pyro.core.getProxyForURI(uri)
else:
self.dispatcher = _pyro.Proxy(uri)
self.dispatcher._pyroTimeout = 10
if not self.dispatcher.register_worker(self.WORKERNAME):
if using_pyro4:
self.dispatcher._pyroRelease()
else:
self.dispatcher._release()
self.dispatcher = None
else:
break
except _connection_problem:
self.dispatcher = None
if self.dispatcher is not None:
if using_pyro4:
self.dispatcher._pyroTimeout = None
break
sleep_interval = random.uniform(5.0, 15.0)
print("Worker failed to find dispatcher object from "
"name server after %d attempts and %5.2f seconds "
"- trying again in %5.2f seconds." %
(i + 1, cumulative_sleep_time, sleep_interval))
time.sleep(sleep_interval)
cumulative_sleep_time += sleep_interval
if self.dispatcher is None:
raise RuntimeError(
"Worker could not find dispatcher object - giving up")
# There is no need to retain the proxy connection to the
# nameserver, so free up resources on the nameserver thread
URI = None
if using_pyro4:
self.ns._pyroRelease()
URI = self.dispatcher._pyroUri
else:
self.ns._release()
URI = self.dispatcher.URI
print("Connection to dispatch server %s established "
"after %d attempts and %5.2f seconds - "
"this is worker: %s" %
(URI, i + 1, cumulative_sleep_time, self.WORKERNAME))
def DispatcherServer(group=":PyUtilibServer",
daemon_host=None,
daemon_port=0,
nameserver_host=None,
nameserver_port=None,
verbose=False,
max_allowed_connections=None,
worker_limit=None,
clear_group=True):
set_maxconnections(max_allowed_connections=max_allowed_connections)
#
# main program
#
ns = get_nameserver(host=nameserver_host,
port=nameserver_port,
caller_name="Dispatcher")
if clear_group:
for name, uri in get_dispatchers(group=group, ns=ns):
print("Multiple dispatchers not allowed.")
print("dispatch_srvr is shutting down...")
return 1
if using_pyro3:
daemon = _pyro.core.Daemon(host=daemon_host, port=daemon_port)
daemon.useNameServer(ns)
else:
daemon = _pyro.Daemon(host=daemon_host, port=daemon_port)
if using_pyro3:
try:
ns.createGroup(group)
except _pyro.errors.NamingError:
pass
try:
ns.createGroup(group + ".dispatcher")
except _pyro.errors.NamingError:
pass
if clear_group:
try:
ns.unregister(group + ".dispatcher")
except _pyro.errors.NamingError:
pass
else:
if clear_group:
try:
ns.remove(group + ".dispatcher")
except _pyro.errors.NamingError:
pass
disp = Dispatcher(verbose=verbose, worker_limit=worker_limit)
proxy_name = group + ".dispatcher." + str(uuid.uuid4())
if using_pyro3:
uri = daemon.connect(disp, proxy_name)
else:
uri = daemon.register(disp, proxy_name)
ns.register(proxy_name, uri)
# There is no need to retain the proxy connection to the
# nameserver, so free up resources on the nameserver thread
if using_pyro4:
ns._pyroRelease()
else:
ns._release()
print("Dispatcher is ready.")
return daemon.requestLoop()
def __init__(self,
group=":PyUtilibServer",
host=None,
port=None,
num_dispatcher_tries=30,
caller_name="Task Worker",
verbose=False,
name=None):
self._verbose = verbose
# A worker can set this flag
# if an error occurs during processing
self._worker_error = False
self._worker_shutdown = False
self._worker_task_return_queue = _worker_task_return_queue_unset
# sets whether or not multiple tasks should
# be gathered from the worker queue during
# each request for work
self._bulk_task_collection = False
if _pyro is None:
raise ImportError("Pyro or Pyro4 is not available")
# Deprecated in Pyro3
# Removed in Pyro4
if using_pyro3:
_pyro.core.initClient()
if name is None:
self.WORKERNAME = "Worker_%d@%s" % (os.getpid(),
socket.gethostname())
else:
self.WORKERNAME = name
self.ns = get_nameserver(host=host, port=port, caller_name=caller_name)
if self.ns is None:
raise RuntimeError("TaskWorkerBase failed to locate "
"Pyro name server on the network!")
print('Worker attempting to find Pyro dispatcher object...')
cumulative_sleep_time = 0.0
self.dispatcher = None
for i in xrange(0, num_dispatcher_tries):
dispatchers = get_dispatchers(group=group, ns=self.ns)
random.shuffle(dispatchers)
for name, uri in dispatchers:
try:
if using_pyro3:
self.dispatcher = _pyro.core.getProxyForURI(uri)
else:
self.dispatcher = _pyro.Proxy(uri)
self.dispatcher._pyroTimeout = 10
if not self.dispatcher.register_worker(self.WORKERNAME):
if using_pyro4:
self.dispatcher._pyroRelease()
else:
self.dispatcher._release()
self.dispatcher = None
else:
break
except _connection_problem:
self.dispatcher = None
if self.dispatcher is not None:
if using_pyro4:
self.dispatcher._pyroTimeout = None
break
sleep_interval = random.uniform(5.0, 15.0)
print("Worker failed to find dispatcher object from "
"name server after %d attempts and %5.2f seconds "
"- trying again in %5.2f seconds." %
(i + 1, cumulative_sleep_time, sleep_interval))
time.sleep(sleep_interval)
cumulative_sleep_time += sleep_interval
if self.dispatcher is None:
raise RuntimeError(
"Worker could not find dispatcher object - giving up")
# There is no need to retain the proxy connection to the
# nameserver, so free up resources on the nameserver thread
URI = None
if using_pyro4:
self.ns._pyroRelease()
URI = self.dispatcher._pyroUri
else:
self.ns._release()
URI = self.dispatcher.URI
print("Connection to dispatch server %s established "
"after %d attempts and %5.2f seconds - "
"this is worker: %s" %
(URI, i + 1, cumulative_sleep_time, self.WORKERNAME))
def __init__(self,
group=":PyUtilibServer",
type=None,
host=None,
port=None,
num_dispatcher_tries=30,
caller_name="Client",
dispatcher=None):
if _pyro is None:
raise ImportError("Pyro or Pyro4 is not available")
self.type = type
self.id = 0
# Deprecated in Pyro3
# Removed in Pyro4
if using_pyro3:
_pyro.core.initClient()
self.ns = None
self.CLIENTNAME = "%d@%s" % (os.getpid(), socket.gethostname())
self.dispatcher = None
if dispatcher is None:
self.ns = get_nameserver(host=host,
port=port,
caller_name=caller_name)
if self.ns is None:
raise RuntimeError("Client failed to locate Pyro name "
"server on the network!")
print('Client attempting to find Pyro dispatcher object...')
self.URI = None
cumulative_sleep_time = 0.0
for i in xrange(0, num_dispatcher_tries):
dispatchers = pyutilib.pyro.util.get_dispatchers(ns=self.ns)
for (name, uri) in dispatchers:
self.URI = uri
print("Dispatcher Object URI: " + str(self.URI))
break
if self.URI is not None:
break
sleep_interval = 10.0
print("Client failed to find dispatcher object from name "
"server after %d attempts and %5.2f seconds - trying "
"again in %5.2f seconds." %
(i + 1, cumulative_sleep_time, sleep_interval))
time.sleep(sleep_interval)
cumulative_sleep_time += sleep_interval
if self.URI is None:
print('Client could not find dispatcher object - giving up')
raise SystemExit
# connect to the dispatcher
if using_pyro3:
self.dispatcher = _pyro.core.getProxyForURI(self.URI)
else:
self.dispatcher = _pyro.Proxy(self.URI)
print("Connection to dispatch server established after %d "
"attempts and %5.2f seconds - this is client: %s" %
(i + 1, cumulative_sleep_time, self.CLIENTNAME))
# There is no need to retain the proxy connection to the
# nameserver, so free up resources on the nameserver thread
if using_pyro4:
self.ns._pyroRelease()
else:
self.ns._release()
else:
assert port is None
assert host is None
self.dispatcher = dispatcher
if using_pyro4:
self.URI = self.dispatcher._pyroUri
else:
self.URI = self.dispatcher.URI
print('Client assigned dispatcher with URI=%s' % (self.URI))
def DispatcherServer(group=":PyUtilibServer",
daemon_host=None,
daemon_port=0,
nameserver_host=None,
nameserver_port=None,
verbose=False,
max_allowed_connections=None,
worker_limit=None,
clear_group=True):
set_maxconnections(max_allowed_connections=max_allowed_connections)
#
# main program
#
ns = get_nameserver(
host=nameserver_host, port=nameserver_port, caller_name="Dispatcher")
if clear_group:
for name, uri in get_dispatchers(group=group, ns=ns):
print("Multiple dispatchers not allowed.")
print("dispatch_srvr is shutting down...")
return 1
if using_pyro3:
daemon = _pyro.core.Daemon(host=daemon_host, port=daemon_port)
daemon.useNameServer(ns)
else:
daemon = _pyro.Daemon(host=daemon_host, port=daemon_port)
if using_pyro3:
try:
ns.createGroup(group)
except _pyro.errors.NamingError:
pass
try:
ns.createGroup(group + ".dispatcher")
except _pyro.errors.NamingError:
pass
if clear_group:
try:
ns.unregister(group + ".dispatcher")
except _pyro.errors.NamingError:
pass
else:
if clear_group:
try:
ns.remove(group + ".dispatcher")
except _pyro.errors.NamingError:
pass
disp = Dispatcher(verbose=verbose, worker_limit=worker_limit)
proxy_name = group + ".dispatcher." + str(uuid.uuid4())
if using_pyro3:
uri = daemon.connect(disp, proxy_name)
else:
uri = daemon.register(disp, proxy_name)
ns.register(proxy_name, uri)
# There is no need to retain the proxy connection to the
# nameserver, so free up resources on the nameserver thread
if using_pyro4:
ns._pyroRelease()
else:
ns._release()
print("Dispatcher is ready.")
return daemon.requestLoop()
def __init__(self,
group=":PyUtilibServer",
type=None,
host=None,
port=None,
num_dispatcher_tries=30,
caller_name="Client",
dispatcher=None):
if _pyro is None:
raise ImportError("Pyro or Pyro4 is not available")
self.type = type
self.id = 0
# Deprecated in Pyro3
# Removed in Pyro4
if using_pyro3:
_pyro.core.initClient()
self.ns = None
self.CLIENTNAME = "%d@%s" % (os.getpid(), socket.gethostname())
self.dispatcher = None
if dispatcher is None:
self.ns = get_nameserver(
host=host, port=port, caller_name=caller_name)
if self.ns is None:
raise RuntimeError("Client failed to locate Pyro name "
"server on the network!")
print('Client attempting to find Pyro dispatcher object...')
self.URI = None
cumulative_sleep_time = 0.0
for i in xrange(0, num_dispatcher_tries):
dispatchers = pyutilib.pyro.util.get_dispatchers(ns=self.ns)
for (name, uri) in dispatchers:
self.URI = uri
print("Dispatcher Object URI: " + str(self.URI))
break
if self.URI is not None:
break
sleep_interval = 10.0
print("Client failed to find dispatcher object from name "
"server after %d attempts and %5.2f seconds - trying "
"again in %5.2f seconds." %
(i + 1, cumulative_sleep_time, sleep_interval))
time.sleep(sleep_interval)
cumulative_sleep_time += sleep_interval
if self.URI is None:
print('Client could not find dispatcher object - giving up')
raise SystemExit
# connect to the dispatcher
if using_pyro3:
self.dispatcher = _pyro.core.getProxyForURI(self.URI)
else:
self.dispatcher = _pyro.Proxy(self.URI)
print("Connection to dispatch server established after %d "
"attempts and %5.2f seconds - this is client: %s" %
(i + 1, cumulative_sleep_time, self.CLIENTNAME))
# There is no need to retain the proxy connection to the
# nameserver, so free up resources on the nameserver thread
if using_pyro4:
self.ns._pyroRelease()
else:
self.ns._release()
else:
assert port is None
assert host is None
self.dispatcher = dispatcher
if using_pyro4:
self.URI = self.dispatcher._pyroUri
else:
self.URI = self.dispatcher.URI
print('Client assigned dispatcher with URI=%s' % (self.URI))
