import os
import string
import six
from taskflow.utils import misc
from taskflow import version
BANNER_HEADER = string.Template("""
___ __
| |_
|ask |low v$version
""".strip())
BANNER_HEADER = BANNER_HEADER.substitute(version=version.version_string())
def make_banner(what, chapters):
"""Makes a taskflow banner string.
For example::
>>> from taskflow.utils import banner
>>> chapters = {
'Connection details': {
'Topic': 'hello',
},
'Powered by': {
'Executor': 'parallel',
},
Supported endpoints:$endpoints
System details:
Hostname = $hostname
Pid = $pid
Platform = $platform
Python = $python
Thread id = $thread_id
""".strip())
BANNER_TEMPLATE.defaults = {
# These values may not be possible to fetch/known, default to unknown...
'pid': '???',
'hostname': '???',
'executor_thread_count': '???',
'endpoints': ' %s' % ([]),
# These are static (avoid refetching...)
'version': version.version_string(),
'python': sys.version.split("\n", 1)[0].strip(),
}
LOG = logging.getLogger(__name__)
class Worker(object):
"""Worker that can be started on a remote host for handling tasks requests.
:param url: broker url
:param exchange: broker exchange name
:param topic: topic name under which worker is stated
:param tasks: task list that worker is capable of performing, items in
the list can be one of the following types; 1, a string naming the
python module name to search for tasks in or the task class name; 2, a
# The master toctree document. master_doc = 'index' # General information about the project. project = u'taskflow Release Notes' copyright = u'2016, taskflow Developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. from taskflow import version as taskflow_version # The full version, including alpha/beta/rc tags. release = taskflow_version.version_string() # The short X.Y version. version = release # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files.
class Worker(object):
"""Worker that can be started on a remote host for handling tasks requests.
:param url: broker url
:param exchange: broker exchange name
:param topic: topic name under which worker is stated
:param tasks: task list that worker is capable of performing, items in
the list can be one of the following types; 1, a string naming the
python module name to search for tasks in or the task class name; 2, a
python module to search for tasks in; 3, a task class object that
will be used to create tasks from.
:param executor: custom executor object that can used for processing
requests in separate threads (if not provided one will be created)
:param threads_count: threads count to be passed to the
default executor (used only if an executor is not
passed in)
:param transport: transport to be used (e.g. amqp, memory, etc.)
:param transport_options: transport specific options (see:
http://kombu.readthedocs.org/ for what these
options imply and are expected to be)
:param retry_options: retry specific options
(see: :py:attr:`~.proxy.Proxy.DEFAULT_RETRY_OPTIONS`)
"""
BANNER_TEMPLATE = string.Template("""
TaskFlow v${version} WBE worker.
Connection details:
Driver = $transport_driver
Exchange = $exchange
Topic = $topic
Transport = $transport_type
Uri = $connection_uri
Powered by:
Executor = $executor_type
Thread count = $executor_thread_count
Supported endpoints:$endpoints
System details:
Hostname = $hostname
Pid = $pid
Platform = $platform
Python = $python
Thread id = $thread_id
""".strip())
# See: http://bugs.python.org/issue13173 for why we are doing this...
BANNER_TEMPLATE.defaults = {
# These values may not be possible to fetch/known, default
# to ??? to represent that they are unknown...
'pid': '???',
'hostname': '???',
'executor_thread_count': '???',
'endpoints': ' %s' % ([]),
# These are static (avoid refetching...)
'version': version.version_string(),
'python': sys.version.split("\n", 1)[0].strip(),
}
def __init__(self,
exchange,
topic,
tasks,
executor=None,
threads_count=None,
url=None,
transport=None,
transport_options=None,
retry_options=None):
self._topic = topic
self._executor = executor
self._owns_executor = False
if self._executor is None:
self._executor = futurist.ThreadPoolExecutor(
max_workers=threads_count)
self._owns_executor = True
self._endpoints = self._derive_endpoints(tasks)
self._exchange = exchange
self._server = server.Server(topic,
exchange,
self._executor,
self._endpoints,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
@staticmethod
def _derive_endpoints(tasks):
"""Derive endpoints from list of strings, classes or packages."""
derived_tasks = misc.find_subclasses(tasks, t_task.BaseTask)
return [endpoint.Endpoint(task) for task in derived_tasks]
def _generate_banner(self):
"""Generates a banner that can be useful to display before running."""
try:
tpl_params = dict(self.BANNER_TEMPLATE.defaults)
except AttributeError:
tpl_params = {}
connection_details = self._server.connection_details
transport = connection_details.transport
if transport.driver_version:
transport_driver = "%s v%s" % (transport.driver_name,
transport.driver_version)
else:
transport_driver = transport.driver_name
tpl_params['transport_driver'] = transport_driver
tpl_params['exchange'] = self._exchange
tpl_params['topic'] = self._topic
tpl_params['transport_type'] = transport.driver_type
tpl_params['connection_uri'] = connection_details.uri
tpl_params['executor_type'] = reflection.get_class_name(self._executor)
threads_count = getattr(self._executor, 'max_workers', None)
if threads_count is not None:
tpl_params['executor_thread_count'] = threads_count
if self._endpoints:
pretty_endpoints = []
for ep in self._endpoints:
pretty_endpoints.append(" - %s" % ep)
# This ensures there is a newline before the list...
tpl_params['endpoints'] = "\n" + "\n".join(pretty_endpoints)
try:
tpl_params['hostname'] = socket.getfqdn()
except socket.error:
pass
try:
tpl_params['pid'] = os.getpid()
except OSError:
pass
tpl_params['platform'] = platform.platform()
tpl_params['thread_id'] = tu.get_ident()
banner = self.BANNER_TEMPLATE.substitute(**tpl_params)
# NOTE(harlowja): this is needed since the template in this file
# will always have newlines that end with '\n' (even on different
# platforms due to the way this source file is encoded) so we have
# to do this little dance to make it platform neutral...
return misc.fix_newlines(banner)
def run(self, display_banner=True, banner_writer=None):
"""Runs the worker."""
if display_banner:
banner = self._generate_banner()
if banner_writer is None:
for line in banner.splitlines():
LOG.info(line)
else:
banner_writer(banner)
self._server.start()
def wait(self):
"""Wait until worker is started."""
self._server.wait()
def stop(self):
"""Stop worker."""
self._server.stop()
if self._owns_executor:
self._executor.shutdown()
# under the License.
import os
import string
import six
from taskflow.utils import misc
from taskflow import version
BANNER_HEADER = string.Template("""
___ __
| |_
|ask |low v$version
""".strip())
BANNER_HEADER = BANNER_HEADER.substitute(version=version.version_string())
def make_banner(what, chapters):
"""Makes a taskflow banner string.
For example::
>>> from taskflow.utils import banner
>>> chapters = {
'Connection details': {
'Topic': 'hello',
},
'Powered by': {
'Executor': 'parallel',
},
Supported endpoints:$endpoints
System details:
Hostname = $hostname
Pid = $pid
Platform = $platform
Python = $python
Thread id = $thread_id
""".strip())
BANNER_TEMPLATE.defaults = {
# These values may not be possible to fetch/known, default to unknown...
'pid': '???',
'hostname': '???',
'executor_thread_count': '???',
'endpoints': ' %s' % ([]),
# These are static (avoid refetching...)
'version': version.version_string(),
'python': sys.version.split("\n", 1)[0].strip(),
}
LOG = logging.getLogger(__name__)
class Worker(object):
"""Worker that can be started on a remote host for handling tasks requests.
:param url: broker url
:param exchange: broker exchange name
:param topic: topic name under which worker is stated
:param tasks: task list that worker is capable of performing, items in
the list can be one of the following types; 1, a string naming the
python module name to search for tasks in or the task class name; 2, a
