def parse(self, cmdline_args=None, main_task_cls=None):
parser = PassThroughOptionParser()
def add_task_option(p):
if main_task_cls:
p.add_option('--task', help='Task to run (one of ' + Register.tasks_str() + ') [default: %default]', default=main_task_cls.task_family)
else:
p.add_option('--task', help='Task to run (one of %s)' % Register.tasks_str())
add_global_parameters(parser, optparse=True)
add_task_option(parser)
options, args = parser.parse_args(args=cmdline_args)
task_cls_name = options.task
if self.__existing_optparse:
parser = self.__existing_optparse
else:
parser = optparse.OptionParser()
add_task_option(parser)
task_cls = Register.get_task_cls(task_cls_name)
# Register all parameters as a big mess
add_global_parameters(parser, optparse=True)
add_task_parameters(parser, task_cls, optparse=True)
# Parse and run
options, args = parser.parse_args(args=cmdline_args)
set_global_parameters(options)
task_params = get_task_parameters(task_cls, options)
return [task_cls(**task_params)]
def __init__(self, *args, **kwargs) -> None:
super(PandasTypeConfigMap, self).__init__(*args, **kwargs)
task_names = Register.task_names()
task_classes = [Register.get_task_cls(task_name) for task_name in task_names]
self._map = {
task_class.task_namespace: task_class
for task_class in task_classes if issubclass(task_class, PandasTypeConfig) and task_class != PandasTypeConfig
}
def test_externalize_taskclass(self):
with self.assertRaises(TaskClassNotFoundException):
Register.get_task_cls('scooby.Doo')
class Task1(luigi.Task):
@classmethod
def get_task_family(cls):
return "scooby.Doo"
self.assertEqual(Task1, Register.get_task_cls('scooby.Doo'))
class Task2(luigi.Task):
@classmethod
def get_task_family(cls):
return "scooby.Doo"
with self.assertRaises(TaskClassAmbigiousException):
Register.get_task_cls('scooby.Doo')
class Task3(luigi.Task):
@classmethod
def get_task_family(cls):
return "scooby.Doo"
# There previously was a rare bug where the third installed class could
# "undo" class ambiguity.
with self.assertRaises(TaskClassAmbigiousException):
Register.get_task_cls('scooby.Doo')
def test_externalize_taskclass(self):
with self.assertRaises(TaskClassNotFoundException):
Register.get_task_cls('scooby.Doo')
class Task1(luigi.Task):
@classmethod
def get_task_family(cls):
return "scooby.Doo"
self.assertEqual(Task1, Register.get_task_cls('scooby.Doo'))
class Task2(luigi.Task):
@classmethod
def get_task_family(cls):
return "scooby.Doo"
with self.assertRaises(TaskClassAmbigiousException):
Register.get_task_cls('scooby.Doo')
class Task3(luigi.Task):
@classmethod
def get_task_family(cls):
return "scooby.Doo"
# There previously was a rare bug where the third installed class could
# "undo" class ambiguity.
with self.assertRaises(TaskClassAmbigiousException):
Register.get_task_cls('scooby.Doo')
def __init__(self, cmdline_args):
"""
Initialize cmd line args
"""
known_args, _ = self._build_parser().parse_known_args(args=cmdline_args)
self._attempt_load_module(known_args)
# We have to parse again now. As the positionally first unrecognized
# argument (the task) could be different.
known_args, _ = self._build_parser().parse_known_args(args=cmdline_args)
root_task = known_args.root_task
parser = self._build_parser(root_task=root_task,
help_all=known_args.core_help_all)
self._possibly_exit_with_help(parser, known_args)
if not root_task:
raise SystemExit('No task specified')
else:
# Check that what we believe to be the task is correctly spelled
Register.get_task_cls(root_task)
known_args = parser.parse_args(args=cmdline_args)
self.known_args = known_args # Also publicly expose parsed arguments
def parse_task(self, cmdline_args=None, main_task_cls=None):
if cmdline_args is None:
cmdline_args = sys.argv[1:]
parser = argparse.ArgumentParser()
add_global_parameters(parser)
if main_task_cls:
add_task_parameters(parser, main_task_cls)
args = parser.parse_args(args=cmdline_args)
task_cls = main_task_cls
else:
task_names = Register.task_names()
# Parse global arguments and pull out the task name.
# We used to do this using subparsers+command, but some issues with
# argparse across different versions of Python (2.7.9) made it hard.
args, unknown = parser.parse_known_args(
args=[a for a in cmdline_args if a != '--help'])
if len(unknown) == 0:
# In case it included a --help argument, run again
parser.parse_known_args(args=cmdline_args)
raise SystemExit('No task specified')
task_name = unknown[0]
if task_name not in task_names:
error_task_names(task_name, task_names)
task_cls = Register.get_task_cls(task_name)
# Add a subparser to parse task-specific arguments
subparsers = parser.add_subparsers(dest='command')
subparser = subparsers.add_parser(task_name)
# Add both task and global params here so that we can support both:
# test.py --global-param xyz Test --n 42
# test.py Test --n 42 --global-param xyz
add_global_parameters(subparser)
add_task_parameters(subparser, task_cls)
# Workaround for bug in argparse for Python 2.7.9
# See https://mail.python.org/pipermail/python-dev/2015-January/137699.html
subargs = parser.parse_args(args=cmdline_args)
for key, value in vars(subargs).items():
if value: # Either True (for boolean args) or non-None (everything else)
setattr(args, key, value)
# Notice that this is not side effect free because it might set global params
set_global_parameters(args)
task_params = get_task_parameters(task_cls, args)
return [task_cls(**task_params)]
def __init__(self, cmdline_args):
"""
Initialize cmd line args
"""
known_args, _ = self._build_parser().parse_known_args(args=cmdline_args)
self._attempt_load_module(known_args)
# We have to parse again now. As the positionally first unrecognized
# argument (the task) could be different.
known_args, _ = self._build_parser().parse_known_args(args=cmdline_args)
root_task = known_args.root_task
parser = self._build_parser(root_task=root_task,
help_all=known_args.core_help_all)
self._possibly_exit_with_help(parser, known_args)
if not root_task:
raise SystemExit('No task specified')
else:
# Check that what we believe to be the task is correctly spelled
Register.get_task_cls(root_task)
known_args = parser.parse_args(args=cmdline_args)
self.known_args = known_args # Also publically expose parsed arguments
def parse_task(self, cmdline_args=None, main_task_cls=None):
if cmdline_args is None:
cmdline_args = sys.argv[1:]
parser = argparse.ArgumentParser()
add_global_parameters(parser)
if main_task_cls:
add_task_parameters(parser, main_task_cls)
args = parser.parse_args(args=cmdline_args)
task_cls = main_task_cls
else:
task_names = Register.task_names()
# Parse global arguments and pull out the task name.
# We used to do this using subparsers+command, but some issues with
# argparse across different versions of Python (2.7.9) made it hard.
args, unknown = parser.parse_known_args(args=[a for a in cmdline_args if a != '--help'])
if len(unknown) == 0:
# In case it included a --help argument, run again
parser.parse_known_args(args=cmdline_args)
raise SystemExit('No task specified')
task_name = unknown[0]
if task_name not in task_names:
error_task_names(task_name, task_names)
task_cls = Register.get_task_cls(task_name)
# Add a subparser to parse task-specific arguments
subparsers = parser.add_subparsers(dest='command')
subparser = subparsers.add_parser(task_name)
# Add both task and global params here so that we can support both:
# test.py --global-param xyz Test --n 42
# test.py Test --n 42 --global-param xyz
add_global_parameters(subparser)
add_task_parameters(subparser, task_cls)
# Workaround for bug in argparse for Python 2.7.9
# See https://mail.python.org/pipermail/python-dev/2015-January/137699.html
subargs = parser.parse_args(args=cmdline_args)
for key, value in vars(subargs).items():
if value: # Either True (for boolean args) or non-None (everything else)
setattr(args, key, value)
# Notice that this is not side effect free because it might set global params
set_global_parameters(args)
task_params = get_task_parameters(task_cls, args)
return [task_cls(**task_params)]
def parse(self, cmdline_args=None, main_task_cls=None):
parser = PassThroughOptionParser()
def add_task_option(p):
if main_task_cls:
p.add_option('--task',
help='Task to run (one of ' +
Register.tasks_str() + ') [default: %default]',
default=main_task_cls.task_family)
else:
p.add_option('--task',
help='Task to run (one of %s)' %
Register.tasks_str())
add_global_parameters(parser, optparse=True)
add_task_option(parser)
options, args = parser.parse_args(args=cmdline_args)
task_cls_name = options.task
if self.__existing_optparse:
parser = self.__existing_optparse
else:
parser = optparse.OptionParser()
add_task_option(parser)
task_cls = Register.get_task_cls(task_cls_name)
# Register all parameters as a big mess
add_global_parameters(parser, optparse=True)
add_task_parameters(parser, task_cls, optparse=True)
# Parse and run
options, args = parser.parse_args(args=cmdline_args)
set_global_parameters(options)
task_params = get_task_parameters(task_cls, options)
return [task_cls(**task_params)]
def get_task_cls(mod_cls):
""" Resolve string to task class object
Reuse Luigi's service that registers task types
Like Luigi, we assume that the mod_cls is 'module.class' and we assume
that the user has put there pipe location on PYTHONPATH.
:param mod_cls: '<module>.<class>'
:return: Task class object
"""
from luigi.task_register import Register
mod_path = mod_cls.split('.')
mod = '.'.join(mod_path[:-1])
cls = mod_path[-1]
if mod is not None:
__import__(mod)
task_cls = Register.get_task_cls(cls)
return task_cls
def _get_task_cls(self):
"""
Get the task class
"""
return Register.get_task_cls(self.known_args.root_task)
def get_task_cls(self):
"""
Get the task class
"""
return Register.get_task_cls(self._task_name)
def get_task_cls(self):
"""
Get the task class
"""
return Register.get_task_cls(self._task_name)
def _get_task_cls(self):
"""
Get the task class
"""
return Register.get_task_cls(self.known_args.root_task)
