def execute(args): """ Executes the *config* subprogram with parsed commandline *args*. """ # just print the file location? if args.location: print(Config.instance().config_file) return # every option below requires the name to be set if not args.name: abort( "please give the name of the config in the format <section>[.<option>]" ) # removal if args.remove: abort("config removal not yet implemented") # setting if args.value: abort("config setting not yet implemented") # getting print(get_config(args.name, expand=args.expand))
def execute(args): """ Executes the *config* subprogram with parsed commandline *args*. """ cfg = Config.instance() # just print the file location? if args.location: print(cfg.config_file) return # print sections when none is given if not args.name: print("\n".join(cfg.sections())) return # print section options when none is given section, option = args.name.split( ".", 1) if "." in args.name else (args.name, None) if not option: print("\n".join(cfg.options(section))) return # removal if args.remove: abort("config removal not yet implemented") # setting if args.value: abort("config setting not yet implemented") # getting print( cfg.get_default(section, option, expand_vars=args.expand, expand_user=args.expand))
def execute(args, argv): """ Executes the *run* subprogram with parsed commandline *args*. """ task_family = None error = None # try to infer the task module from the passed task family and import it parts = args.task_family.rsplit(".", 1) if len(parts) == 2: modid, cls_name = parts try: mod = __import__(modid, globals(), locals(), [cls_name]) task_cls = getattr(mod, cls_name, None) if task_cls is not None: if not issubclass(task_cls, Task): abort("object '{}' is not a Task".format(args.task_family)) task_family = task_cls.get_task_family() except ImportError as e: # distinguish import errors resulting from an unknown modid from all other cases modid_unknown = str(e) == "No module named {}".format(modid) if modid_unknown: # keep the error in case the task family cannot be inferred from the index file error = e else: raise # read task info from the index file and import it if task_family is None: cfg = Config.instance() index_file = cfg.get_expanded("core", "index_file") if os.path.exists(index_file): info = read_task_from_index(args.task_family, index_file) if not info: abort("task family '{}' not found in index".format( args.task_family)) modid, task_family, _ = info __import__(modid, globals(), locals()) # complain when no task could be found if task_family is None: if error: raise error else: abort("task '{}' not found".format(args.task_family)) # run luigi sys.argv[0] += " run" luigi_run([task_family] + argv[3:])
def execute(args): """ Executes the *run* subprogram with parsed commandline *args*. """ task_family = None error = None # try to infer the task module from the passed task family and import it parts = args.task_family.rsplit(".", 1) if len(parts) == 2: modid, cls_name = parts try: mod = __import__(modid, globals(), locals(), [cls_name]) if hasattr(mod, cls_name): task_cls = getattr(mod, cls_name) if not issubclass(task_cls, Task): abort("object '{}' is not a Task".format(args.task_family)) task_family = task_cls.task_family except ImportError as e: logger.debug("import error in module {}: {}".format(modid, e)) error = e # read task info from the db file and import it if task_family is None: db_file = Config.instance().get_expanded("core", "db_file") if os.path.exists(db_file): info = read_task_from_db(args.task_family, db_file) if not info: abort("task family '{}' not found in db".format( args.task_family)) modid, task_family, _ = info __import__(modid, globals(), locals()) # complain when no task could be found if task_family is None: if error: raise error else: abort("task '{}' not found".format(args.task_family)) # import the module and run luigi luigi_run([task_family] + sys.argv[3:])
def execute(args): """ Executes the *index* subprogram with parsed commandline *args*. """ cfg = Config.instance() index_file = cfg.get_expanded("core", "index_file") # just print the file location? if args.location: print(index_file) return # just show the file content? if args.show: if os.path.exists(index_file): with open(index_file, "r") as f: print(f.read()) return else: abort("index file {} does not exist".format(index_file)) # just remove the index file? if args.remove: if os.path.exists(index_file): os.remove(index_file) print("removed index file {}".format(index_file)) return # get modules to lookup lookup = [m.strip() for m in cfg.options("modules")] if args.modules: lookup += args.modules print("indexing tasks in {} module(s)".format(len(lookup))) # loop through modules, import everything to load tasks for modid in lookup: if not modid: continue if args.verbose: sys.stdout.write("loading module '{}'".format(modid)) try: import_module(modid) except Exception as e: if not args.verbose: print("error in module '{}': {}".format( colored(modid, "red"), str(e))) else: print("\n\nerror in module '{}':".format(colored(modid, "red"))) traceback.print_exc() continue if args.verbose: print(", {}".format(colored("done", style="bright"))) # determine tasks to write into the index file seen_families = [] task_classes = [] lookup = [Task] while lookup: cls = lookup.pop(0) lookup.extend(cls.__subclasses__()) # skip already seen task families task_family = cls.get_task_family() if task_family in seen_families: continue seen_families.append(task_family) # skip when explicitly excluded if cls.exclude_index: continue # skip external tasks is_external_task = issubclass(cls, ExternalTask) if args.no_externals and is_external_task: continue # skip non-external tasks without run implementation run_is_callable = callable(getattr(cls, "run", None)) run_is_abstract = getattr(cls.run, "__isabstractmethod__", False) if not is_external_task and (not run_is_callable or run_is_abstract): continue # show an error when there is a "-" in the task family as the luigi command line parser will # automatically map it to "_", i.e., it will fail to lookup the actual task class # skip the task if "-" in task_family: logger.critical( "skipping task '{}' as its family '{}' contains a '-' which cannot be " "interpreted by luigi's command line parser, please use '_' or alike" .format(cls, task_family)) continue # show an error when there is a "_" after a "." in the task family, i.e., when there is a # "_" in the class name (which is bad python practice anyway), as the shell autocompletion # is not able to decide whether it should complete the task family or a task-level parameter # skip the task if "_" in task_family.rsplit(".", 1)[-1]: logger.error( "skipping task '{}' as its family '{}' contains a '_' after the namespace " "definition which would lead to ambiguities between task families and task-level " "parameters in the law shell autocompletion".format( cls, task_family)) continue task_classes.append(cls) def get_task_params(cls): params = [] for attr in dir(cls): member = getattr(cls, attr) if isinstance(member, luigi.Parameter): exclude = getattr(cls, "exclude_params_index", set()) if not multi_match(attr, exclude, any): params.append(attr.replace("_", "-")) return params def index_line(cls, params): # format: "module_id:task_family:param param ..." return "{}:{}:{}".format(cls.__module__, cls.get_task_family(), " ".join(params)) stats = OrderedDict() # write the index file if not os.path.exists(os.path.dirname(index_file)): os.makedirs(os.path.dirname(index_file)) with open(index_file, "w") as f: for cls in task_classes: # get prams params = get_task_params(cls) # fill stats if cls.__module__ not in stats: stats[cls.__module__] = [] stats[cls.__module__].append((cls.get_task_family(), params)) f.write(index_line(cls, params) + "\n") # print stats if args.verbose: for mod, data in six.iteritems(stats): print("\nmodule '{}', {} task(s):".format( colored(mod, style="bright"), len(data))) for task_family, _ in data: print(" - {}".format(colored(task_family, "green"))) print("") print("written {} task(s) to index file '{}'".format( len(task_classes), index_file))
def execute(args): if args.value is None and not args.remove: # just print the value print(get_config(args.name)) else: abort("config setting and removal not yet implemented")