def requires(self):
from luigi.task import Register
if not self.task in Register.get_reg().keys():
logger.warn("No such task {} in registry; skipping".format(self.task))
return []
else:
cls = Register.get_reg()[self.task]
return [cls(target=x) for x in self.generic_wrapper_target]
def requires(self):
from luigi.task import Register
if not self.task in Register.get_reg().keys():
logger.warn("No such task {} in registry; skipping".format(
self.task))
return []
else:
cls = Register.get_reg()[self.task]
return [cls(target=x) for x in self.generic_wrapper_target]
def parse_task(self, cmdline_args=None, main_task_cls=None):
parser = argparse.ArgumentParser()
add_global_parameters(parser)
if main_task_cls:
add_task_parameters(parser, main_task_cls)
args = parser.parse_args(args=cmdline_args)
task_cls = main_task_cls
else:
task_names = sorted(Register.get_reg().keys())
# Parse global arguments and pull out the task name.
# We used to do this using subparsers+command, but some issues with
# argparse across different versions of Python (2.7.9) made it hard.
args, unknown = parser.parse_known_args(args=cmdline_args)
if len(unknown) == 0:
raise SystemExit('No task specified')
task_name = unknown[0]
if task_name not in task_names:
error_task_names(task_name, task_names)
task_cls = Register.get_task_cls(task_name)
# Add a subparser to parse task-specific arguments
subparsers = parser.add_subparsers(dest='command')
subparser = subparsers.add_parser(task_name)
# Add both task and global params here so that we can support both:
# test.py --global-param xyz Test --n 42
# test.py Test --n 42 --global-param xyz
add_global_parameters(subparser)
add_task_parameters(subparser, task_cls)
# Workaround for bug in argparse for Python 2.7.9
# See https://mail.python.org/pipermail/python-dev/2015-January/137699.html
subargs = parser.parse_args(args=cmdline_args)
for key, value in vars(subargs).items():
if value: # Either True (for boolean args) or non-None (everything else)
setattr(args, key, value)
# Notice that this is not side effect free because it might set global params
set_global_parameters(args)
task_params = get_task_parameters(task_cls, args)
return [task_cls(**task_params)]
def test_cmdline(self):
# Exposes issue where wrapped tasks are registered twice under
# the same name
from luigi.task import Register
self.assertEqual(Register.get_reg().get('SubtaskDelegator', None), SubtaskDelegator)
def test_cmdline(self):
# Exposes issue where wrapped tasks are registered twice under
# the same name
from luigi.task import Register
self.assertEqual(Register.get_reg().get('SubtaskDelegator', None),
SubtaskDelegator)
def gen_sphinx_tasks(entry_point, labels, *_args, **kwargs):
"""
Writes a file per label, suitable for use by sphinx.ext.autodoc,
using the classes found from entry_point.
Also generates toctree.inc, which can be included from the index
page to provide links to each generated file.
"""
# Declare file header strings
warning = '''.. WARNING: DO NOT EDIT THIS FILE DIRECTLY
Generated by sphinx_source/gen_tasks.py on {now}
'''.format(now=time.strftime('%c'))
toctree_header = '''{warning}
.. toctree::
:maxdepth: 1
'''
incfile_header = '''{warning}
.. _{category_slug}:
Back to :doc:`index`
{label_heading}
'''
# Load modules into memory
stevedore.ExtensionManager(entry_point)
# Used to filter the classes under entry_point
entry_point_dot = '{entry_point}.'.format(entry_point=entry_point)
# Generate a list of output file arguments from the given labels and categories
output = []
categories = kwargs.get('categories', [])
for idx, label in enumerate(labels):
try:
category = ''
if idx < len(categories):
category = categories[idx]
# Create a category slug for sphinx, and name the file with it
category_slug = category.replace(' ', '_') or 'all'
file_name = '{slug}.rst'.format(slug=category_slug)
file_path = os.path.join(SPHINX_DIR, file_name)
file_pointer = open(file_path, "w")
output.append({
'fp':
file_pointer,
'file_name':
file_name,
'category':
category,
'category_slug':
category_slug,
'label':
label,
'label_heading':
"{label}\n{_}".format(label=label, _='=' * len(label)),
'modules': {},
})
except IOError:
sys.exit(
'Unable to write to {file_path}'.format(file_path=file_path))
# Write the header to the table of contents file
tocfile_name = os.path.join(SPHINX_DIR, 'toctree.rst')
try:
tocfile = open(tocfile_name, "w")
tocfile.write(toctree_header.format(warning=warning))
except IOError:
sys.exit(
'Unable to write to {file_name}'.format(file_name=tocfile_name))
# For each Task, sorted by class name
tasks = Register.get_reg()
for name in sorted(tasks):
cls = tasks[name]
module = cls.__module__
# Show only tasks under entry_point
if module.startswith(entry_point_dot):
for out in output:
# Show only tasks in the output category
if getattr(cls, 'task_category', '') == out['category']:
if module not in out['modules']:
out['modules'][module] = {}
out['modules'][module][name] = cls
for out in output:
modules = sorted(out['modules'].keys())
if modules:
tocfile.write("\n {incfile}".format(incfile=out['file_name']))
out['fp'].write(incfile_header.format(warning=warning, **out))
for module in modules:
# Strip off entry_point to avoid redundancy in documentation
module_heading = '{module}'.format(
module=module.replace(entry_point_dot, ''))
out['fp'].write("\n\n{module_heading}\n{_}".format(
module_heading=module_heading, _='-' * len(module_heading)))
out['fp'].write(
"\n\n.. automodule:: {module}".format(module=module))
names = out['modules'][module]
for name in sorted(names):
out['fp'].write("\n\n.. autoclass:: {name}".format(name=name))
out['fp'].close()
tocfile.close()
def gen_sphinx_tasks(entry_point, labels, *_args, **kwargs):
"""
Writes a file per label, suitable for use by sphinx.ext.autodoc,
using the classes found from entry_point.
Also generates toctree.inc, which can be included from the index
page to provide links to each generated file.
"""
# Declare file header strings
warning = '''.. WARNING: DO NOT EDIT THIS FILE DIRECTLY
Generated by sphinx_source/gen_tasks.py on {now}
'''.format(now=time.strftime('%c'))
toctree_header = '''{warning}
:orphan:
.. toctree::
:maxdepth: 1
'''
incfile_header = '''{warning}
.. _{category_slug}:
Back to :doc:`index`
{label_heading}
'''
# Load modules into memory
stevedore.ExtensionManager(entry_point)
# Used to filter the classes under entry_point
entry_point_dot = '{entry_point}.'.format(entry_point=entry_point)
# Generate a list of output file arguments from the given labels and categories
output = []
categories = kwargs.get('categories', [])
for idx, label in enumerate(labels):
try:
category = ''
if idx < len(categories):
category = categories[idx]
# Create a category slug for sphinx, and name the file with it
category_slug = category.replace(' ', '_') or 'all'
file_name = '{slug}.rst'.format(slug=category_slug)
file_path = os.path.join(SPHINX_DIR, file_name)
file_pointer = open(file_path, "w")
output.append({
'fp': file_pointer,
'file_name': file_name,
'category': category,
'category_slug': category_slug,
'label': label,
'label_heading': "{label}\n{_}".format(label=label, _='=' * len(label)),
'modules': {},
})
except IOError:
sys.exit('Unable to write to {file_path}'.format(file_path=file_path))
# Write the header to the table of contents file
tocfile_name = os.path.join(SPHINX_DIR, 'toctree.rst')
try:
tocfile = open(tocfile_name, "w")
tocfile.write(toctree_header.format(warning=warning))
except IOError:
sys.exit('Unable to write to {file_name}'.format(file_name=tocfile_name))
# For each Task, sorted by class name
tasks = Register.get_reg()
for name in sorted(tasks):
cls = tasks[name]
for out in output:
# Show only tasks under entry_point
module = cls.__module__
if module.startswith(entry_point_dot):
# Strip off entry_point to avoid redundancy in documentation
module = module.replace(entry_point_dot, '')
if getattr(cls, 'task_category', '') == out['category']:
if module not in out['modules']:
out['modules'][module] = {}
out['modules'][module][name] = cls
for out in output:
modules = sorted(out['modules'].keys())
if modules:
tocfile.write("\n {incfile}".format(incfile=out['file_name']))
out['fp'].write(incfile_header.format(warning=warning, **out))
for module in modules:
module_heading = '{module}'.format(module=module)
out['fp'].write("\n\n{module_heading}\n{_}".format(
module_heading=module_heading, _='-' * len(module_heading)))
out['fp'].write("\n\n.. automodule:: {module}".format(module=module))
names = out['modules'][module]
for name in sorted(names):
out['fp'].write("\n\n.. autoclass:: {name}".format(name=name))
out['fp'].close()
tocfile.close()
