def requires(self): from luigi.task import Register if not self.task in Register.get_reg().keys(): logger.warn("No such task {} in registry; skipping".format(self.task)) return [] else: cls = Register.get_reg()[self.task] return [cls(target=x) for x in self.generic_wrapper_target]
def requires(self): from luigi.task import Register if not self.task in Register.get_reg().keys(): logger.warn("No such task {} in registry; skipping".format( self.task)) return [] else: cls = Register.get_reg()[self.task] return [cls(target=x) for x in self.generic_wrapper_target]
def parse_task(self, cmdline_args=None, main_task_cls=None): parser = argparse.ArgumentParser() add_global_parameters(parser) if main_task_cls: add_task_parameters(parser, main_task_cls) args = parser.parse_args(args=cmdline_args) task_cls = main_task_cls else: task_names = sorted(Register.get_reg().keys()) # Parse global arguments and pull out the task name. # We used to do this using subparsers+command, but some issues with # argparse across different versions of Python (2.7.9) made it hard. args, unknown = parser.parse_known_args(args=cmdline_args) if len(unknown) == 0: raise SystemExit('No task specified') task_name = unknown[0] if task_name not in task_names: error_task_names(task_name, task_names) task_cls = Register.get_task_cls(task_name) # Add a subparser to parse task-specific arguments subparsers = parser.add_subparsers(dest='command') subparser = subparsers.add_parser(task_name) # Add both task and global params here so that we can support both: # test.py --global-param xyz Test --n 42 # test.py Test --n 42 --global-param xyz add_global_parameters(subparser) add_task_parameters(subparser, task_cls) # Workaround for bug in argparse for Python 2.7.9 # See https://mail.python.org/pipermail/python-dev/2015-January/137699.html subargs = parser.parse_args(args=cmdline_args) for key, value in vars(subargs).items(): if value: # Either True (for boolean args) or non-None (everything else) setattr(args, key, value) # Notice that this is not side effect free because it might set global params set_global_parameters(args) task_params = get_task_parameters(task_cls, args) return [task_cls(**task_params)]
def test_cmdline(self): # Exposes issue where wrapped tasks are registered twice under # the same name from luigi.task import Register self.assertEqual(Register.get_reg().get('SubtaskDelegator', None), SubtaskDelegator)
def gen_sphinx_tasks(entry_point, labels, *_args, **kwargs): """ Writes a file per label, suitable for use by sphinx.ext.autodoc, using the classes found from entry_point. Also generates toctree.inc, which can be included from the index page to provide links to each generated file. """ # Declare file header strings warning = '''.. WARNING: DO NOT EDIT THIS FILE DIRECTLY Generated by sphinx_source/gen_tasks.py on {now} '''.format(now=time.strftime('%c')) toctree_header = '''{warning} .. toctree:: :maxdepth: 1 ''' incfile_header = '''{warning} .. _{category_slug}: Back to :doc:`index` {label_heading} ''' # Load modules into memory stevedore.ExtensionManager(entry_point) # Used to filter the classes under entry_point entry_point_dot = '{entry_point}.'.format(entry_point=entry_point) # Generate a list of output file arguments from the given labels and categories output = [] categories = kwargs.get('categories', []) for idx, label in enumerate(labels): try: category = '' if idx < len(categories): category = categories[idx] # Create a category slug for sphinx, and name the file with it category_slug = category.replace(' ', '_') or 'all' file_name = '{slug}.rst'.format(slug=category_slug) file_path = os.path.join(SPHINX_DIR, file_name) file_pointer = open(file_path, "w") output.append({ 'fp': file_pointer, 'file_name': file_name, 'category': category, 'category_slug': category_slug, 'label': label, 'label_heading': "{label}\n{_}".format(label=label, _='=' * len(label)), 'modules': {}, }) except IOError: sys.exit( 'Unable to write to {file_path}'.format(file_path=file_path)) # Write the header to the table of contents file tocfile_name = os.path.join(SPHINX_DIR, 'toctree.rst') try: tocfile = open(tocfile_name, "w") tocfile.write(toctree_header.format(warning=warning)) except IOError: sys.exit( 'Unable to write to {file_name}'.format(file_name=tocfile_name)) # For each Task, sorted by class name tasks = Register.get_reg() for name in sorted(tasks): cls = tasks[name] module = cls.__module__ # Show only tasks under entry_point if module.startswith(entry_point_dot): for out in output: # Show only tasks in the output category if getattr(cls, 'task_category', '') == out['category']: if module not in out['modules']: out['modules'][module] = {} out['modules'][module][name] = cls for out in output: modules = sorted(out['modules'].keys()) if modules: tocfile.write("\n {incfile}".format(incfile=out['file_name'])) out['fp'].write(incfile_header.format(warning=warning, **out)) for module in modules: # Strip off entry_point to avoid redundancy in documentation module_heading = '{module}'.format( module=module.replace(entry_point_dot, '')) out['fp'].write("\n\n{module_heading}\n{_}".format( module_heading=module_heading, _='-' * len(module_heading))) out['fp'].write( "\n\n.. automodule:: {module}".format(module=module)) names = out['modules'][module] for name in sorted(names): out['fp'].write("\n\n.. autoclass:: {name}".format(name=name)) out['fp'].close() tocfile.close()
def gen_sphinx_tasks(entry_point, labels, *_args, **kwargs): """ Writes a file per label, suitable for use by sphinx.ext.autodoc, using the classes found from entry_point. Also generates toctree.inc, which can be included from the index page to provide links to each generated file. """ # Declare file header strings warning = '''.. WARNING: DO NOT EDIT THIS FILE DIRECTLY Generated by sphinx_source/gen_tasks.py on {now} '''.format(now=time.strftime('%c')) toctree_header = '''{warning} :orphan: .. toctree:: :maxdepth: 1 ''' incfile_header = '''{warning} .. _{category_slug}: Back to :doc:`index` {label_heading} ''' # Load modules into memory stevedore.ExtensionManager(entry_point) # Used to filter the classes under entry_point entry_point_dot = '{entry_point}.'.format(entry_point=entry_point) # Generate a list of output file arguments from the given labels and categories output = [] categories = kwargs.get('categories', []) for idx, label in enumerate(labels): try: category = '' if idx < len(categories): category = categories[idx] # Create a category slug for sphinx, and name the file with it category_slug = category.replace(' ', '_') or 'all' file_name = '{slug}.rst'.format(slug=category_slug) file_path = os.path.join(SPHINX_DIR, file_name) file_pointer = open(file_path, "w") output.append({ 'fp': file_pointer, 'file_name': file_name, 'category': category, 'category_slug': category_slug, 'label': label, 'label_heading': "{label}\n{_}".format(label=label, _='=' * len(label)), 'modules': {}, }) except IOError: sys.exit('Unable to write to {file_path}'.format(file_path=file_path)) # Write the header to the table of contents file tocfile_name = os.path.join(SPHINX_DIR, 'toctree.rst') try: tocfile = open(tocfile_name, "w") tocfile.write(toctree_header.format(warning=warning)) except IOError: sys.exit('Unable to write to {file_name}'.format(file_name=tocfile_name)) # For each Task, sorted by class name tasks = Register.get_reg() for name in sorted(tasks): cls = tasks[name] for out in output: # Show only tasks under entry_point module = cls.__module__ if module.startswith(entry_point_dot): # Strip off entry_point to avoid redundancy in documentation module = module.replace(entry_point_dot, '') if getattr(cls, 'task_category', '') == out['category']: if module not in out['modules']: out['modules'][module] = {} out['modules'][module][name] = cls for out in output: modules = sorted(out['modules'].keys()) if modules: tocfile.write("\n {incfile}".format(incfile=out['file_name'])) out['fp'].write(incfile_header.format(warning=warning, **out)) for module in modules: module_heading = '{module}'.format(module=module) out['fp'].write("\n\n{module_heading}\n{_}".format( module_heading=module_heading, _='-' * len(module_heading))) out['fp'].write("\n\n.. automodule:: {module}".format(module=module)) names = out['modules'][module] for name in sorted(names): out['fp'].write("\n\n.. autoclass:: {name}".format(name=name)) out['fp'].close() tocfile.close()