Ejemplo n.º 1
0
    def __init__(self,
                 group=":PyUtilibServer",
                 host=None,
                 port=None,
                 num_dispatcher_tries=30,
                 caller_name="Task Worker",
                 verbose=False,
                 name=None):

        self._verbose = verbose
        # A worker can set this flag
        # if an error occurs during processing
        self._worker_error = False
        self._worker_shutdown = False
        self._worker_task_return_queue = _worker_task_return_queue_unset
        # sets whether or not multiple tasks should
        # be gathered from the worker queue during
        # each request for work
        self._bulk_task_collection = False

        if _pyro is None:
            raise ImportError("Pyro or Pyro4 is not available")

        # Deprecated in Pyro3
        # Removed in Pyro4
        if using_pyro3:
            _pyro.core.initClient()

        if name is None:
            self.WORKERNAME = "Worker_%d@%s" % (os.getpid(),
                                                socket.gethostname())
        else:
            self.WORKERNAME = name

        self.ns = get_nameserver(host=host, port=port, caller_name=caller_name)
        if self.ns is None:
            raise RuntimeError("TaskWorkerBase failed to locate "
                               "Pyro name server on the network!")
        print('Worker attempting to find Pyro dispatcher object...')
        cumulative_sleep_time = 0.0
        self.dispatcher = None
        for i in xrange(0, num_dispatcher_tries):
            dispatchers = get_dispatchers(group=group, ns=self.ns)
            random.shuffle(dispatchers)
            for name, uri in dispatchers:
                try:
                    if using_pyro3:
                        self.dispatcher = _pyro.core.getProxyForURI(uri)
                    else:
                        self.dispatcher = _pyro.Proxy(uri)
                        self.dispatcher._pyroTimeout = 10
                    if not self.dispatcher.register_worker(self.WORKERNAME):
                        if using_pyro4:
                            self.dispatcher._pyroRelease()
                        else:
                            self.dispatcher._release()
                        self.dispatcher = None
                    else:
                        break
                except _connection_problem:
                    self.dispatcher = None
            if self.dispatcher is not None:
                if using_pyro4:
                    self.dispatcher._pyroTimeout = None
                break
            sleep_interval = random.uniform(5.0, 15.0)
            print("Worker failed to find dispatcher object from "
                  "name server after %d attempts and %5.2f seconds "
                  "- trying again in %5.2f seconds." %
                  (i + 1, cumulative_sleep_time, sleep_interval))
            time.sleep(sleep_interval)
            cumulative_sleep_time += sleep_interval

        if self.dispatcher is None:
            raise RuntimeError(
                "Worker could not find dispatcher object - giving up")

        # There is no need to retain the proxy connection to the
        # nameserver, so free up resources on the nameserver thread
        URI = None
        if using_pyro4:
            self.ns._pyroRelease()
            URI = self.dispatcher._pyroUri
        else:
            self.ns._release()
            URI = self.dispatcher.URI

        print("Connection to dispatch server %s established "
              "after %d attempts and %5.2f seconds - "
              "this is worker: %s" %
              (URI, i + 1, cumulative_sleep_time, self.WORKERNAME))
Ejemplo n.º 2
0
def DispatcherServer(group=":PyUtilibServer",
                     daemon_host=None,
                     daemon_port=0,
                     nameserver_host=None,
                     nameserver_port=None,
                     verbose=False,
                     max_allowed_connections=None,
                     worker_limit=None,
                     clear_group=True):

    set_maxconnections(max_allowed_connections=max_allowed_connections)

    #
    # main program
    #
    ns = get_nameserver(host=nameserver_host,
                        port=nameserver_port,
                        caller_name="Dispatcher")

    if clear_group:
        for name, uri in get_dispatchers(group=group, ns=ns):
            print("Multiple dispatchers not allowed.")
            print("dispatch_srvr is shutting down...")
            return 1

    if using_pyro3:
        daemon = _pyro.core.Daemon(host=daemon_host, port=daemon_port)
        daemon.useNameServer(ns)
    else:
        daemon = _pyro.Daemon(host=daemon_host, port=daemon_port)

    if using_pyro3:
        try:
            ns.createGroup(group)
        except _pyro.errors.NamingError:
            pass
        try:
            ns.createGroup(group + ".dispatcher")
        except _pyro.errors.NamingError:
            pass
        if clear_group:
            try:
                ns.unregister(group + ".dispatcher")
            except _pyro.errors.NamingError:
                pass
    else:
        if clear_group:
            try:
                ns.remove(group + ".dispatcher")
            except _pyro.errors.NamingError:
                pass

    disp = Dispatcher(verbose=verbose, worker_limit=worker_limit)
    proxy_name = group + ".dispatcher." + str(uuid.uuid4())
    if using_pyro3:
        uri = daemon.connect(disp, proxy_name)
    else:
        uri = daemon.register(disp, proxy_name)
        ns.register(proxy_name, uri)

    # There is no need to retain the proxy connection to the
    # nameserver, so free up resources on the nameserver thread
    if using_pyro4:
        ns._pyroRelease()
    else:
        ns._release()

    print("Dispatcher is ready.")
    return daemon.requestLoop()
Ejemplo n.º 3
0
    def __init__(self,
                 group=":PyUtilibServer",
                 host=None,
                 port=None,
                 num_dispatcher_tries=30,
                 caller_name="Task Worker",
                 verbose=False,
                 name=None):

        self._verbose = verbose
        # A worker can set this flag
        # if an error occurs during processing
        self._worker_error = False
        self._worker_shutdown = False
        self._worker_task_return_queue = _worker_task_return_queue_unset
        # sets whether or not multiple tasks should
        # be gathered from the worker queue during
        # each request for work
        self._bulk_task_collection = False

        if _pyro is None:
            raise ImportError("Pyro or Pyro4 is not available")

        # Deprecated in Pyro3
        # Removed in Pyro4
        if using_pyro3:
            _pyro.core.initClient()

        if name is None:
            self.WORKERNAME = "Worker_%d@%s" % (os.getpid(),
                                                socket.gethostname())
        else:
            self.WORKERNAME = name

        self.ns = get_nameserver(host=host, port=port, caller_name=caller_name)
        if self.ns is None:
            raise RuntimeError("TaskWorkerBase failed to locate "
                               "Pyro name server on the network!")
        print('Worker attempting to find Pyro dispatcher object...')
        cumulative_sleep_time = 0.0
        self.dispatcher = None
        for i in xrange(0, num_dispatcher_tries):
            dispatchers = get_dispatchers(group=group, ns=self.ns)
            random.shuffle(dispatchers)
            for name, uri in dispatchers:
                try:
                    if using_pyro3:
                        self.dispatcher = _pyro.core.getProxyForURI(uri)
                    else:
                        self.dispatcher = _pyro.Proxy(uri)
                        self.dispatcher._pyroTimeout = 10
                    if not self.dispatcher.register_worker(self.WORKERNAME):
                        if using_pyro4:
                            self.dispatcher._pyroRelease()
                        else:
                            self.dispatcher._release()
                        self.dispatcher = None
                    else:
                        break
                except _connection_problem:
                    self.dispatcher = None
            if self.dispatcher is not None:
                if using_pyro4:
                    self.dispatcher._pyroTimeout = None
                break
            sleep_interval = random.uniform(5.0, 15.0)
            print("Worker failed to find dispatcher object from "
                  "name server after %d attempts and %5.2f seconds "
                  "- trying again in %5.2f seconds." %
                  (i + 1, cumulative_sleep_time, sleep_interval))
            time.sleep(sleep_interval)
            cumulative_sleep_time += sleep_interval

        if self.dispatcher is None:
            raise RuntimeError(
                "Worker could not find dispatcher object - giving up")

        # There is no need to retain the proxy connection to the
        # nameserver, so free up resources on the nameserver thread
        URI = None
        if using_pyro4:
            self.ns._pyroRelease()
            URI = self.dispatcher._pyroUri
        else:
            self.ns._release()
            URI = self.dispatcher.URI

        print("Connection to dispatch server %s established "
              "after %d attempts and %5.2f seconds - "
              "this is worker: %s" %
              (URI, i + 1, cumulative_sleep_time, self.WORKERNAME))
Ejemplo n.º 4
0
    def __init__(self,
                 group=":PyUtilibServer",
                 type=None,
                 host=None,
                 port=None,
                 num_dispatcher_tries=30,
                 caller_name="Client",
                 dispatcher=None):

        if _pyro is None:
            raise ImportError("Pyro or Pyro4 is not available")
        self.type = type
        self.id = 0

        # Deprecated in Pyro3
        # Removed in Pyro4
        if using_pyro3:
            _pyro.core.initClient()

        self.ns = None

        self.CLIENTNAME = "%d@%s" % (os.getpid(), socket.gethostname())
        self.dispatcher = None
        if dispatcher is None:
            self.ns = get_nameserver(host=host,
                                     port=port,
                                     caller_name=caller_name)
            if self.ns is None:
                raise RuntimeError("Client failed to locate Pyro name "
                                   "server on the network!")
            print('Client attempting to find Pyro dispatcher object...')
            self.URI = None
            cumulative_sleep_time = 0.0
            for i in xrange(0, num_dispatcher_tries):

                dispatchers = pyutilib.pyro.util.get_dispatchers(ns=self.ns)

                for (name, uri) in dispatchers:
                    self.URI = uri
                    print("Dispatcher Object URI: " + str(self.URI))
                    break

                if self.URI is not None:
                    break

                sleep_interval = 10.0
                print("Client failed to find dispatcher object from name "
                      "server after %d attempts and %5.2f seconds - trying "
                      "again in %5.2f seconds." %
                      (i + 1, cumulative_sleep_time, sleep_interval))

                time.sleep(sleep_interval)
                cumulative_sleep_time += sleep_interval
            if self.URI is None:
                print('Client could not find dispatcher object - giving up')
                raise SystemExit

            # connect to the dispatcher
            if using_pyro3:
                self.dispatcher = _pyro.core.getProxyForURI(self.URI)
            else:
                self.dispatcher = _pyro.Proxy(self.URI)

            print("Connection to dispatch server established after %d "
                  "attempts and %5.2f seconds - this is client: %s" %
                  (i + 1, cumulative_sleep_time, self.CLIENTNAME))

            # There is no need to retain the proxy connection to the
            # nameserver, so free up resources on the nameserver thread
            if using_pyro4:
                self.ns._pyroRelease()
            else:
                self.ns._release()

        else:
            assert port is None
            assert host is None
            self.dispatcher = dispatcher
            if using_pyro4:
                self.URI = self.dispatcher._pyroUri
            else:
                self.URI = self.dispatcher.URI
            print('Client assigned dispatcher with URI=%s' % (self.URI))
Ejemplo n.º 5
0
def DispatcherServer(group=":PyUtilibServer",
                     daemon_host=None,
                     daemon_port=0,
                     nameserver_host=None,
                     nameserver_port=None,
                     verbose=False,
                     max_allowed_connections=None,
                     worker_limit=None,
                     clear_group=True):

    set_maxconnections(max_allowed_connections=max_allowed_connections)

    #
    # main program
    #
    ns = get_nameserver(
        host=nameserver_host, port=nameserver_port, caller_name="Dispatcher")

    if clear_group:
        for name, uri in get_dispatchers(group=group, ns=ns):
            print("Multiple dispatchers not allowed.")
            print("dispatch_srvr is shutting down...")
            return 1

    if using_pyro3:
        daemon = _pyro.core.Daemon(host=daemon_host, port=daemon_port)
        daemon.useNameServer(ns)
    else:
        daemon = _pyro.Daemon(host=daemon_host, port=daemon_port)

    if using_pyro3:
        try:
            ns.createGroup(group)
        except _pyro.errors.NamingError:
            pass
        try:
            ns.createGroup(group + ".dispatcher")
        except _pyro.errors.NamingError:
            pass
        if clear_group:
            try:
                ns.unregister(group + ".dispatcher")
            except _pyro.errors.NamingError:
                pass
    else:
        if clear_group:
            try:
                ns.remove(group + ".dispatcher")
            except _pyro.errors.NamingError:
                pass

    disp = Dispatcher(verbose=verbose, worker_limit=worker_limit)
    proxy_name = group + ".dispatcher." + str(uuid.uuid4())
    if using_pyro3:
        uri = daemon.connect(disp, proxy_name)
    else:
        uri = daemon.register(disp, proxy_name)
        ns.register(proxy_name, uri)

    # There is no need to retain the proxy connection to the
    # nameserver, so free up resources on the nameserver thread
    if using_pyro4:
        ns._pyroRelease()
    else:
        ns._release()

    print("Dispatcher is ready.")
    return daemon.requestLoop()
Ejemplo n.º 6
0
    def __init__(self,
                 group=":PyUtilibServer",
                 type=None,
                 host=None,
                 port=None,
                 num_dispatcher_tries=30,
                 caller_name="Client",
                 dispatcher=None):

        if _pyro is None:
            raise ImportError("Pyro or Pyro4 is not available")
        self.type = type
        self.id = 0

        # Deprecated in Pyro3
        # Removed in Pyro4
        if using_pyro3:
            _pyro.core.initClient()

        self.ns = None

        self.CLIENTNAME = "%d@%s" % (os.getpid(), socket.gethostname())
        self.dispatcher = None
        if dispatcher is None:
            self.ns = get_nameserver(
                host=host, port=port, caller_name=caller_name)
            if self.ns is None:
                raise RuntimeError("Client failed to locate Pyro name "
                                   "server on the network!")
            print('Client attempting to find Pyro dispatcher object...')
            self.URI = None
            cumulative_sleep_time = 0.0
            for i in xrange(0, num_dispatcher_tries):

                dispatchers = pyutilib.pyro.util.get_dispatchers(ns=self.ns)

                for (name, uri) in dispatchers:
                    self.URI = uri
                    print("Dispatcher Object URI: " + str(self.URI))
                    break

                if self.URI is not None:
                    break

                sleep_interval = 10.0
                print("Client failed to find dispatcher object from name "
                      "server after %d attempts and %5.2f seconds - trying "
                      "again in %5.2f seconds." %
                      (i + 1, cumulative_sleep_time, sleep_interval))

                time.sleep(sleep_interval)
                cumulative_sleep_time += sleep_interval
            if self.URI is None:
                print('Client could not find dispatcher object - giving up')
                raise SystemExit

            # connect to the dispatcher
            if using_pyro3:
                self.dispatcher = _pyro.core.getProxyForURI(self.URI)
            else:
                self.dispatcher = _pyro.Proxy(self.URI)

            print("Connection to dispatch server established after %d "
                  "attempts and %5.2f seconds - this is client: %s" %
                  (i + 1, cumulative_sleep_time, self.CLIENTNAME))

            # There is no need to retain the proxy connection to the
            # nameserver, so free up resources on the nameserver thread
            if using_pyro4:
                self.ns._pyroRelease()
            else:
                self.ns._release()

        else:
            assert port is None
            assert host is None
            self.dispatcher = dispatcher
            if using_pyro4:
                self.URI = self.dispatcher._pyroUri
            else:
                self.URI = self.dispatcher.URI
            print('Client assigned dispatcher with URI=%s' % (self.URI))