def __critical_abort(self, code: int): print_idx(self.idx, "Collecting files in working directory") archive = shutil.make_archive( os.path.join( os.getcwd(), 'sltx-log-' + su.get_now() + '-' + su.sanitize_filename(self.file)), 'zip', self.__f("{out_dir}")) print_idx( self.idx, " - Created: \"" + archive + "\" (" + os.path.basename(archive) + ")") # We have to force latexmk into think it has to re-run # TODO: We should check if not aux but we do this # We do this as we need a change and the percent itself gets consumed # This ensures a different sequence every time that will be deleted # on a successful run if sg.configuration[sg.C_CLEAN_ON_FAILURE]: LOGGER.info('Running auto clean on failure.') sg.args.exclude_patterns = [] sg.args.include_patterns = [self.__f("{out_dir}")] scmd.cleanse_caches() else: with open(self.__f("{out_dir}/{file_base_noext}.aux"), 'a') as f: f.write('%% sltx errormark' + str(random.random()) + " - " + str(random.random())) # automatic analyze os.system('sltx analyze "' + archive + '"') raise rex.RecipeException( archive, 'Recipe for ' + str(self.idx) + ' failed with code: ' + str(code) + '. See logfile: \"' + archive + "\"")
def grab_from(idx: str, path: str, data: dict, target: str, key: str, grabber, extras: list) -> bool: if key not in data: if len(extras) == 0: print_idx(idx, " ! Key '" + key + "' not found. Won't grab any...") return False else: data[key] = extras else: data[key].extend(extras) grabs = [] for grab_pattern in set(data[key]): cur_grab_pattern = split_grab_pattern(grab_pattern, target) # maybe forbid level up? grabs.extend( map(lambda x, pattern=cur_grab_pattern: (x, pattern[1]), glob.glob(os.path.join(path, cur_grab_pattern[0]), recursive=True))) # extra so i can setup installer afterwards more easily print_idx( idx, " > Grabbing the following for installation: " + str([os.path.relpath(f[0], path) for f in grabs])) for grab in grabs: grabber(grab, target, path) return True
def write_proc_to_log(idx: str, stream, mirror: bool): while True: line = stream.readline() if not line: break line_utf8 = line.decode('utf-8') write_to_log(line_utf8) if mirror: print_idx(idx, line_utf8)
def extend_grab_from_local(idx: str, driver_target_dir: str, data: dict) -> Tuple[list, list]: """May install extra profiles This method will check for a local dep file, and if present check for a profiles key. if present it will check for a selected profile in the data dict. If so, select it, if none, set the default Args: idx (str): index to use in multithreading driver_target_dir (str): target dir of the current driver data (str): data dict for local configuration Returns: (list,list) - a list of additional dependencies in the format 'files, folder' """ if 'dep' not in data: dep = sg.DEFAULT_DEPENDENCY else: dep = data['dep'] dep_files = glob.glob(os.path.join(driver_target_dir, dep), recursive=True) if len(dep_files) <= 0: return [], [] # load file and check for default # TODO: avoid reloading if recursive? file_profiles = {} for dep_file in dep_files: file_profiles = load_dependencies_config(dep_file, file_profiles) if 'profiles' not in file_profiles: return [], [] file_profiles = file_profiles['profiles'] # Note: I do want always a default profile so it makes live easier for me if 'default' not in file_profiles: raise DependencyProfileException( 'No default profile for enlisted profiles. Found: ' + str(file_profiles)) if 'profile' in data: requested_profile = data['profile'] if requested_profile not in file_profiles: raise DependencyProfileException('Requested profile (' + requested_profile + ') not found. Found: ' + str(file_profiles)) else: requested_profile = 'default' added_profiles: dict = file_profiles[requested_profile] print_idx( idx, ' > Loaded profile (' + requested_profile + '): ' + str(added_profiles)) # TODO: this may be beautified grab_files = added_profiles[ 'grab-files'] if 'grab-files' in added_profiles else "" grab_dirs = added_profiles[ 'grab_dirs'] if 'grab_dirs' in added_profiles else "" return grab_files, grab_dirs
def __save_files(self, our_dir: str): """Retrieves the resulting files by patterns Args: our_dir (str): the path to the target directory for caught files """ print_idx(self.idx, '> Retrieving resulting files to "' + our_dir + '"') got_files = [] for wf in self.settings['wanted_files'] + sg.configuration[ sg.C_WANTED_FILES]: wf = self.__f(wf) if sg.args.verbose: print_idx(self.idx, ' - Retrieving files for pattern "' + wf + '"') wanted = glob.glob( os.path.join(sg.configuration[sg.C_WORKING_DIR], wf)) for f in wanted: if sg.args.verbose: print_idx(self.idx, "Saving \"" + f + "\" ") shutil.copy2(f, our_dir) got_files += wanted if not sg.args.verbose: print_idx(self.idx, "Saved files (" + str(got_files) + ")")
def detect_driver(idx: str, url: str) -> str: """Tries to match the given patterns to [description] This could be optimized by pre-compile the given patterns. Args: idx (str): the current index number for logging url (str): the url to adapt the driver from Returns: (str): The driver key to use """ print_idx(idx, " - Auto-detecting driver...") for key, patterns in sg.configuration[C_DRIVER_PATTERNS].items(): for pattern in patterns: if re.search(pattern, url): return key print_idx(idx, " ! No driver found...") sys.exit(1)
def recursive_dependencies(idx: str, driver_target_dir: str, data: dict, dep_name: str, target: str): if 'dep' not in data: print_idx( idx, "No 'dep' key found for dep: " + dep_name + " using the default (" + sg.DEFAULT_DEPENDENCY + ")") data['dep'] = sg.DEFAULT_DEPENDENCY dep_files = glob.glob(os.path.join(driver_target_dir, data['dep']), recursive=True) print_idx(idx, " - Found dep-config: " + str(dep_files)) if len(dep_files) <= 0: return new_dependencies = {} for dep_file in dep_files: new_dependencies = load_dependencies_config(dep_file, new_dependencies) _install_dependencies(idx, new_dependencies, target)
def __init__(self, recipe_path: str, file: str, idx: str): super().__init__() self.file = file self.idx = idx recipe_full_path = recipe_path if not os.path.isfile(recipe_full_path): recipe_full_path = str( files(sltxpkg.data.recipes).joinpath(recipe_path)) if not os.path.isfile(recipe_full_path): print_idx( self.idx, "Recipe " + recipe_full_path + " was not found. Exiting.") exit(1) print_idx(self.idx, "Loading recipe: " + recipe_full_path) y_conf = su.load_yaml(recipe_full_path) self.settings = {**self.settings, **y_conf} if sg.args.quiet: self.quiet = self.settings['quiet'] self.__process_tools() self.__sanitize_extra_args() # we need them as a single string
def grab_stuff(idx: str, dep_name: str, target_dir: str, data: dict, target: str): extra_files, extra_dirs = extend_grab_from_local(idx, target_dir, data) print_idx(idx, " > Grabbing dependencies for " + dep_name) print_idx(idx, " - Grabby-Grab-Grab files from \"" + target_dir + "\"...") got_files = grab_from(idx, target_dir, data, target, 'grab-files', f_grab_files, extra_files) print_idx(idx, " - Grabby-Grab-Grab dirs from \"" + target_dir + "\"...") got_dirs = grab_from(idx, target_dir, data, target, 'grab-dirs', f_grab_dirs, extra_dirs) if not got_files and not got_dirs: print_idx(idx, " ! No grabs performed!") write_to_log("No grabs performed for: " + dep_name)
def use_driver(idx: str, data: dict, dep_name: str, driver: str, target: str): # default no arguments if "args" not in data: data["args"] = "" driver_data = sg.configuration[C_DRIVERS][driver] command = driver_data["command"].format(**data, **sg.configuration, dep_name=dep_name) driver_target_dir = get_target_dir(data, dep_name, driver) if driver_data["needs-delete"] and os.path.isdir(driver_target_dir): print_idx( idx, " - Target folder " + driver_target_dir + "exists. Will be deleted as the driver needs this") shutil.rmtree(driver_target_dir) if driver_data["needs-create"] and not os.path.isdir(driver_target_dir): print_idx(idx, " - Target folder " + driver_target_dir + " needs creation") os.makedirs(driver_target_dir) print_idx(idx, " > Executing: " + command) with Popen(command, stdout=PIPE, stderr=PIPE, shell=True) as feedback: return_code = feedback.wait() write_proc_to_log(idx, feedback.stdout, False) if return_code != 0: print_idx(idx, " - Error-Log of Driver:") write_proc_to_log(idx, feedback.stderr, return_code != 0) if sg.configuration[C_RECURSIVE]: recursive_dependencies(idx, driver_target_dir, data, dep_name, target) if return_code != 0: print_idx(idx, " ! Driver failed with code" + str(feedback) + "exiting.") sys.exit(return_code) grab_stuff(idx, dep_name, driver_target_dir, data, target)
def __runcmds(self, cmds: List[str]): for cmd in cmds: cmd = self.__f(cmd) # expand if sg.args.verbose: print_idx(self.idx, " - " + cmd) os.system(cmd)
def run(self): """Executes the configured Recipe """ print_idx(self.idx, "Processing file: " + self.file, pre='\n') sc.assure_dirs() # Ensure Working diSr and texmf home sc.assure_dir('file cache', self.__f("{out_dir}"), True) print_idx(self.idx, self.__f("> Running recipe \"{name}\" by \"{author}\".")) self.__runhooks('pre') print_idx( self.idx, "> Running the compile commands (" + str(len(self.settings['run'])) + ")") for i, cmd in enumerate(self.settings['run']): cmd = self.__f(cmd) # expand if sg.args.verbose: print_idx(self.idx, " - " + cmd) fback = os.system(cmd) if fback != 0: print_idx( self.idx, "\033[31m ! The command failed. Initiating critical abort...\033[m" ) self.__critical_abort(fback) self.__runhooks('in') self.__save_files(os.getcwd()) self.__runhooks('post') if sg.configuration[sg.C_CLEANUP]: print_idx(self.idx, "> Cleaning up (configured by configuration)") self.__runcmds(self.settings['cleanup_cmds'])
def __runhooks(self, hookid: str): if sg.args.verbose: print_idx(self.idx, "> Hooks for \"" + hookid + "\"") self.__runcmds(self.settings['hooks'][hookid])
def install_dependency(name: str, idx: str, data: dict, target: str): print_idx(idx, 'Loading "' + name + '"') if "url" not in data: print_idx(idx, " ! The dependency did not have an url-tag attached") exit(1) url = data["url"] print_idx(idx, ' - Loading from: "' + url + '"') if "driver" not in data: if not sg.configuration[C_AUTODETECT_DRIVERS]: print_idx(idx, " ! No driver given and auto-detection disabled!") else: data["driver"] = detect_driver(idx, url) driver = data["driver"] print_idx(idx, " - Using driver: \"" + driver + "\"") if name in loaded: print_idx( idx, " > Skipping retrieval " + name + " as it was already loaded by another dep.") grab_stuff(idx, name, get_target_dir(data, name, driver), data, target) return loaded.append(name) if driver not in sg.configuration[C_DRIVERS]: print_idx( idx, ' ! The selected driver is unknown. Loaded:' + sg.configuration[C_DRIVERS]) sys.exit(2) use_driver(idx, data, name, driver, target)