import os import string import six from taskflow.utils import misc from taskflow import version BANNER_HEADER = string.Template(""" ___ __ | |_ |ask |low v$version """.strip()) BANNER_HEADER = BANNER_HEADER.substitute(version=version.version_string()) def make_banner(what, chapters): """Makes a taskflow banner string. For example:: >>> from taskflow.utils import banner >>> chapters = { 'Connection details': { 'Topic': 'hello', }, 'Powered by': { 'Executor': 'parallel', },
Supported endpoints:$endpoints System details: Hostname = $hostname Pid = $pid Platform = $platform Python = $python Thread id = $thread_id """.strip()) BANNER_TEMPLATE.defaults = { # These values may not be possible to fetch/known, default to unknown... 'pid': '???', 'hostname': '???', 'executor_thread_count': '???', 'endpoints': ' %s' % ([]), # These are static (avoid refetching...) 'version': version.version_string(), 'python': sys.version.split("\n", 1)[0].strip(), } LOG = logging.getLogger(__name__) class Worker(object): """Worker that can be started on a remote host for handling tasks requests. :param url: broker url :param exchange: broker exchange name :param topic: topic name under which worker is stated :param tasks: task list that worker is capable of performing, items in the list can be one of the following types; 1, a string naming the python module name to search for tasks in or the task class name; 2, a
# The master toctree document. master_doc = 'index' # General information about the project. project = u'taskflow Release Notes' copyright = u'2016, taskflow Developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. from taskflow import version as taskflow_version # The full version, including alpha/beta/rc tags. release = taskflow_version.version_string() # The short X.Y version. version = release # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files.
class Worker(object): """Worker that can be started on a remote host for handling tasks requests. :param url: broker url :param exchange: broker exchange name :param topic: topic name under which worker is stated :param tasks: task list that worker is capable of performing, items in the list can be one of the following types; 1, a string naming the python module name to search for tasks in or the task class name; 2, a python module to search for tasks in; 3, a task class object that will be used to create tasks from. :param executor: custom executor object that can used for processing requests in separate threads (if not provided one will be created) :param threads_count: threads count to be passed to the default executor (used only if an executor is not passed in) :param transport: transport to be used (e.g. amqp, memory, etc.) :param transport_options: transport specific options (see: http://kombu.readthedocs.org/ for what these options imply and are expected to be) :param retry_options: retry specific options (see: :py:attr:`~.proxy.Proxy.DEFAULT_RETRY_OPTIONS`) """ BANNER_TEMPLATE = string.Template(""" TaskFlow v${version} WBE worker. Connection details: Driver = $transport_driver Exchange = $exchange Topic = $topic Transport = $transport_type Uri = $connection_uri Powered by: Executor = $executor_type Thread count = $executor_thread_count Supported endpoints:$endpoints System details: Hostname = $hostname Pid = $pid Platform = $platform Python = $python Thread id = $thread_id """.strip()) # See: http://bugs.python.org/issue13173 for why we are doing this... BANNER_TEMPLATE.defaults = { # These values may not be possible to fetch/known, default # to ??? to represent that they are unknown... 'pid': '???', 'hostname': '???', 'executor_thread_count': '???', 'endpoints': ' %s' % ([]), # These are static (avoid refetching...) 'version': version.version_string(), 'python': sys.version.split("\n", 1)[0].strip(), } def __init__(self, exchange, topic, tasks, executor=None, threads_count=None, url=None, transport=None, transport_options=None, retry_options=None): self._topic = topic self._executor = executor self._owns_executor = False if self._executor is None: self._executor = futurist.ThreadPoolExecutor( max_workers=threads_count) self._owns_executor = True self._endpoints = self._derive_endpoints(tasks) self._exchange = exchange self._server = server.Server(topic, exchange, self._executor, self._endpoints, url=url, transport=transport, transport_options=transport_options, retry_options=retry_options) @staticmethod def _derive_endpoints(tasks): """Derive endpoints from list of strings, classes or packages.""" derived_tasks = misc.find_subclasses(tasks, t_task.BaseTask) return [endpoint.Endpoint(task) for task in derived_tasks] def _generate_banner(self): """Generates a banner that can be useful to display before running.""" try: tpl_params = dict(self.BANNER_TEMPLATE.defaults) except AttributeError: tpl_params = {} connection_details = self._server.connection_details transport = connection_details.transport if transport.driver_version: transport_driver = "%s v%s" % (transport.driver_name, transport.driver_version) else: transport_driver = transport.driver_name tpl_params['transport_driver'] = transport_driver tpl_params['exchange'] = self._exchange tpl_params['topic'] = self._topic tpl_params['transport_type'] = transport.driver_type tpl_params['connection_uri'] = connection_details.uri tpl_params['executor_type'] = reflection.get_class_name(self._executor) threads_count = getattr(self._executor, 'max_workers', None) if threads_count is not None: tpl_params['executor_thread_count'] = threads_count if self._endpoints: pretty_endpoints = [] for ep in self._endpoints: pretty_endpoints.append(" - %s" % ep) # This ensures there is a newline before the list... tpl_params['endpoints'] = "\n" + "\n".join(pretty_endpoints) try: tpl_params['hostname'] = socket.getfqdn() except socket.error: pass try: tpl_params['pid'] = os.getpid() except OSError: pass tpl_params['platform'] = platform.platform() tpl_params['thread_id'] = tu.get_ident() banner = self.BANNER_TEMPLATE.substitute(**tpl_params) # NOTE(harlowja): this is needed since the template in this file # will always have newlines that end with '\n' (even on different # platforms due to the way this source file is encoded) so we have # to do this little dance to make it platform neutral... return misc.fix_newlines(banner) def run(self, display_banner=True, banner_writer=None): """Runs the worker.""" if display_banner: banner = self._generate_banner() if banner_writer is None: for line in banner.splitlines(): LOG.info(line) else: banner_writer(banner) self._server.start() def wait(self): """Wait until worker is started.""" self._server.wait() def stop(self): """Stop worker.""" self._server.stop() if self._owns_executor: self._executor.shutdown()
# under the License. import os import string import six from taskflow.utils import misc from taskflow import version BANNER_HEADER = string.Template(""" ___ __ | |_ |ask |low v$version """.strip()) BANNER_HEADER = BANNER_HEADER.substitute(version=version.version_string()) def make_banner(what, chapters): """Makes a taskflow banner string. For example:: >>> from taskflow.utils import banner >>> chapters = { 'Connection details': { 'Topic': 'hello', }, 'Powered by': { 'Executor': 'parallel', },