def execute(args):
"""
Executes the *config* subprogram with parsed commandline *args*.
"""
# just print the file location?
if args.location:
print(Config.instance().config_file)
return
# every option below requires the name to be set
if not args.name:
abort(
"please give the name of the config in the format <section>[.<option>]"
)
# removal
if args.remove:
abort("config removal not yet implemented")
# setting
if args.value:
abort("config setting not yet implemented")
# getting
print(get_config(args.name, expand=args.expand))
def execute(args):
"""
Executes the *config* subprogram with parsed commandline *args*.
"""
cfg = Config.instance()
# just print the file location?
if args.location:
print(cfg.config_file)
return
# print sections when none is given
if not args.name:
print("\n".join(cfg.sections()))
return
# print section options when none is given
section, option = args.name.split(
".", 1) if "." in args.name else (args.name, None)
if not option:
print("\n".join(cfg.options(section)))
return
# removal
if args.remove:
abort("config removal not yet implemented")
# setting
if args.value:
abort("config setting not yet implemented")
# getting
print(
cfg.get_default(section,
option,
expand_vars=args.expand,
expand_user=args.expand))
def execute(args, argv):
"""
Executes the *run* subprogram with parsed commandline *args*.
"""
task_family = None
error = None
# try to infer the task module from the passed task family and import it
parts = args.task_family.rsplit(".", 1)
if len(parts) == 2:
modid, cls_name = parts
try:
mod = __import__(modid, globals(), locals(), [cls_name])
task_cls = getattr(mod, cls_name, None)
if task_cls is not None:
if not issubclass(task_cls, Task):
abort("object '{}' is not a Task".format(args.task_family))
task_family = task_cls.get_task_family()
except ImportError as e:
# distinguish import errors resulting from an unknown modid from all other cases
modid_unknown = str(e) == "No module named {}".format(modid)
if modid_unknown:
# keep the error in case the task family cannot be inferred from the index file
error = e
else:
raise
# read task info from the index file and import it
if task_family is None:
cfg = Config.instance()
index_file = cfg.get_expanded("core", "index_file")
if os.path.exists(index_file):
info = read_task_from_index(args.task_family, index_file)
if not info:
abort("task family '{}' not found in index".format(
args.task_family))
modid, task_family, _ = info
__import__(modid, globals(), locals())
# complain when no task could be found
if task_family is None:
if error:
raise error
else:
abort("task '{}' not found".format(args.task_family))
# run luigi
sys.argv[0] += " run"
luigi_run([task_family] + argv[3:])
def execute(args):
"""
Executes the *run* subprogram with parsed commandline *args*.
"""
task_family = None
error = None
# try to infer the task module from the passed task family and import it
parts = args.task_family.rsplit(".", 1)
if len(parts) == 2:
modid, cls_name = parts
try:
mod = __import__(modid, globals(), locals(), [cls_name])
if hasattr(mod, cls_name):
task_cls = getattr(mod, cls_name)
if not issubclass(task_cls, Task):
abort("object '{}' is not a Task".format(args.task_family))
task_family = task_cls.task_family
except ImportError as e:
logger.debug("import error in module {}: {}".format(modid, e))
error = e
# read task info from the db file and import it
if task_family is None:
db_file = Config.instance().get_expanded("core", "db_file")
if os.path.exists(db_file):
info = read_task_from_db(args.task_family, db_file)
if not info:
abort("task family '{}' not found in db".format(
args.task_family))
modid, task_family, _ = info
__import__(modid, globals(), locals())
# complain when no task could be found
if task_family is None:
if error:
raise error
else:
abort("task '{}' not found".format(args.task_family))
# import the module and run luigi
luigi_run([task_family] + sys.argv[3:])
def execute(args):
"""
Executes the *index* subprogram with parsed commandline *args*.
"""
cfg = Config.instance()
index_file = cfg.get_expanded("core", "index_file")
# just print the file location?
if args.location:
print(index_file)
return
# just show the file content?
if args.show:
if os.path.exists(index_file):
with open(index_file, "r") as f:
print(f.read())
return
else:
abort("index file {} does not exist".format(index_file))
# just remove the index file?
if args.remove:
if os.path.exists(index_file):
os.remove(index_file)
print("removed index file {}".format(index_file))
return
# get modules to lookup
lookup = [m.strip() for m in cfg.options("modules")]
if args.modules:
lookup += args.modules
print("indexing tasks in {} module(s)".format(len(lookup)))
# loop through modules, import everything to load tasks
for modid in lookup:
if not modid:
continue
if args.verbose:
sys.stdout.write("loading module '{}'".format(modid))
try:
import_module(modid)
except Exception as e:
if not args.verbose:
print("error in module '{}': {}".format(
colored(modid, "red"), str(e)))
else:
print("\n\nerror in module '{}':".format(colored(modid,
"red")))
traceback.print_exc()
continue
if args.verbose:
print(", {}".format(colored("done", style="bright")))
# determine tasks to write into the index file
seen_families = []
task_classes = []
lookup = [Task]
while lookup:
cls = lookup.pop(0)
lookup.extend(cls.__subclasses__())
# skip already seen task families
task_family = cls.get_task_family()
if task_family in seen_families:
continue
seen_families.append(task_family)
# skip when explicitly excluded
if cls.exclude_index:
continue
# skip external tasks
is_external_task = issubclass(cls, ExternalTask)
if args.no_externals and is_external_task:
continue
# skip non-external tasks without run implementation
run_is_callable = callable(getattr(cls, "run", None))
run_is_abstract = getattr(cls.run, "__isabstractmethod__", False)
if not is_external_task and (not run_is_callable or run_is_abstract):
continue
# show an error when there is a "-" in the task family as the luigi command line parser will
# automatically map it to "_", i.e., it will fail to lookup the actual task class
# skip the task
if "-" in task_family:
logger.critical(
"skipping task '{}' as its family '{}' contains a '-' which cannot be "
"interpreted by luigi's command line parser, please use '_' or alike"
.format(cls, task_family))
continue
# show an error when there is a "_" after a "." in the task family, i.e., when there is a
# "_" in the class name (which is bad python practice anyway), as the shell autocompletion
# is not able to decide whether it should complete the task family or a task-level parameter
# skip the task
if "_" in task_family.rsplit(".", 1)[-1]:
logger.error(
"skipping task '{}' as its family '{}' contains a '_' after the namespace "
"definition which would lead to ambiguities between task families and task-level "
"parameters in the law shell autocompletion".format(
cls, task_family))
continue
task_classes.append(cls)
def get_task_params(cls):
params = []
for attr in dir(cls):
member = getattr(cls, attr)
if isinstance(member, luigi.Parameter):
exclude = getattr(cls, "exclude_params_index", set())
if not multi_match(attr, exclude, any):
params.append(attr.replace("_", "-"))
return params
def index_line(cls, params):
# format: "module_id:task_family:param param ..."
return "{}:{}:{}".format(cls.__module__, cls.get_task_family(),
" ".join(params))
stats = OrderedDict()
# write the index file
if not os.path.exists(os.path.dirname(index_file)):
os.makedirs(os.path.dirname(index_file))
with open(index_file, "w") as f:
for cls in task_classes:
# get prams
params = get_task_params(cls)
# fill stats
if cls.__module__ not in stats:
stats[cls.__module__] = []
stats[cls.__module__].append((cls.get_task_family(), params))
f.write(index_line(cls, params) + "\n")
# print stats
if args.verbose:
for mod, data in six.iteritems(stats):
print("\nmodule '{}', {} task(s):".format(
colored(mod, style="bright"), len(data)))
for task_family, _ in data:
print(" - {}".format(colored(task_family, "green")))
print("")
print("written {} task(s) to index file '{}'".format(
len(task_classes), index_file))
def execute(args):
if args.value is None and not args.remove:
# just print the value
print(get_config(args.name))
else:
abort("config setting and removal not yet implemented")
