def __critical_abort(self, code: int):
print_idx(self.idx, "Collecting files in working directory")
archive = shutil.make_archive(
os.path.join(
os.getcwd(), 'sltx-log-' + su.get_now() + '-' +
su.sanitize_filename(self.file)), 'zip', self.__f("{out_dir}"))
print_idx(
self.idx, " - Created: \"" + archive + "\" (" +
os.path.basename(archive) + ")")
# We have to force latexmk into think it has to re-run
# TODO: We should check if not aux but we do this
# We do this as we need a change and the percent itself gets consumed
# This ensures a different sequence every time that will be deleted
# on a successful run
if sg.configuration[sg.C_CLEAN_ON_FAILURE]:
LOGGER.info('Running auto clean on failure.')
sg.args.exclude_patterns = []
sg.args.include_patterns = [self.__f("{out_dir}")]
scmd.cleanse_caches()
else:
with open(self.__f("{out_dir}/{file_base_noext}.aux"), 'a') as f:
f.write('%% sltx errormark' + str(random.random()) + " - " +
str(random.random()))
# automatic analyze
os.system('sltx analyze "' + archive + '"')
raise rex.RecipeException(
archive, 'Recipe for ' + str(self.idx) + ' failed with code: ' +
str(code) + '. See logfile: \"' + archive + "\"")
def grab_from(idx: str, path: str, data: dict, target: str, key: str, grabber,
extras: list) -> bool:
if key not in data:
if len(extras) == 0:
print_idx(idx, " ! Key '" + key + "' not found. Won't grab any...")
return False
else:
data[key] = extras
else:
data[key].extend(extras)
grabs = []
for grab_pattern in set(data[key]):
cur_grab_pattern = split_grab_pattern(grab_pattern, target)
# maybe forbid level up?
grabs.extend(
map(lambda x, pattern=cur_grab_pattern: (x, pattern[1]),
glob.glob(os.path.join(path, cur_grab_pattern[0]),
recursive=True)))
# extra so i can setup installer afterwards more easily
print_idx(
idx, " > Grabbing the following for installation: " +
str([os.path.relpath(f[0], path) for f in grabs]))
for grab in grabs:
grabber(grab, target, path)
return True
def write_proc_to_log(idx: str, stream, mirror: bool):
while True:
line = stream.readline()
if not line:
break
line_utf8 = line.decode('utf-8')
write_to_log(line_utf8)
if mirror:
print_idx(idx, line_utf8)
def extend_grab_from_local(idx: str, driver_target_dir: str,
data: dict) -> Tuple[list, list]:
"""May install extra profiles
This method will check for a local dep file, and if present
check for a profiles key. if present it will check for a selected profile in the data dict.
If so, select it, if none, set the default
Args:
idx (str): index to use in multithreading
driver_target_dir (str): target dir of the current driver
data (str): data dict for local configuration
Returns:
(list,list) - a list of additional dependencies in the format 'files, folder'
"""
if 'dep' not in data:
dep = sg.DEFAULT_DEPENDENCY
else:
dep = data['dep']
dep_files = glob.glob(os.path.join(driver_target_dir, dep), recursive=True)
if len(dep_files) <= 0:
return [], []
# load file and check for default
# TODO: avoid reloading if recursive?
file_profiles = {}
for dep_file in dep_files:
file_profiles = load_dependencies_config(dep_file, file_profiles)
if 'profiles' not in file_profiles:
return [], []
file_profiles = file_profiles['profiles']
# Note: I do want always a default profile so it makes live easier for me
if 'default' not in file_profiles:
raise DependencyProfileException(
'No default profile for enlisted profiles. Found: ' +
str(file_profiles))
if 'profile' in data:
requested_profile = data['profile']
if requested_profile not in file_profiles:
raise DependencyProfileException('Requested profile (' +
requested_profile +
') not found. Found: ' +
str(file_profiles))
else:
requested_profile = 'default'
added_profiles: dict = file_profiles[requested_profile]
print_idx(
idx, ' > Loaded profile (' + requested_profile + '): ' +
str(added_profiles))
# TODO: this may be beautified
grab_files = added_profiles[
'grab-files'] if 'grab-files' in added_profiles else ""
grab_dirs = added_profiles[
'grab_dirs'] if 'grab_dirs' in added_profiles else ""
return grab_files, grab_dirs
def __save_files(self, our_dir: str):
"""Retrieves the resulting files by patterns
Args:
our_dir (str): the path to the target directory for caught files
"""
print_idx(self.idx,
'> Retrieving resulting files to "' + our_dir + '"')
got_files = []
for wf in self.settings['wanted_files'] + sg.configuration[
sg.C_WANTED_FILES]:
wf = self.__f(wf)
if sg.args.verbose:
print_idx(self.idx,
' - Retrieving files for pattern "' + wf + '"')
wanted = glob.glob(
os.path.join(sg.configuration[sg.C_WORKING_DIR], wf))
for f in wanted:
if sg.args.verbose:
print_idx(self.idx, "Saving \"" + f + "\" ")
shutil.copy2(f, our_dir)
got_files += wanted
if not sg.args.verbose:
print_idx(self.idx, "Saved files (" + str(got_files) + ")")
def detect_driver(idx: str, url: str) -> str:
"""Tries to match the given patterns to [description]
This could be optimized by pre-compile the given patterns.
Args:
idx (str): the current index number for logging
url (str): the url to adapt the driver from
Returns:
(str): The driver key to use
"""
print_idx(idx, " - Auto-detecting driver...")
for key, patterns in sg.configuration[C_DRIVER_PATTERNS].items():
for pattern in patterns:
if re.search(pattern, url):
return key
print_idx(idx, " ! No driver found...")
sys.exit(1)
def recursive_dependencies(idx: str, driver_target_dir: str, data: dict,
dep_name: str, target: str):
if 'dep' not in data:
print_idx(
idx, "No 'dep' key found for dep: " + dep_name +
" using the default (" + sg.DEFAULT_DEPENDENCY + ")")
data['dep'] = sg.DEFAULT_DEPENDENCY
dep_files = glob.glob(os.path.join(driver_target_dir, data['dep']),
recursive=True)
print_idx(idx, " - Found dep-config: " + str(dep_files))
if len(dep_files) <= 0:
return
new_dependencies = {}
for dep_file in dep_files:
new_dependencies = load_dependencies_config(dep_file, new_dependencies)
_install_dependencies(idx, new_dependencies, target)
def __init__(self, recipe_path: str, file: str, idx: str):
super().__init__()
self.file = file
self.idx = idx
recipe_full_path = recipe_path
if not os.path.isfile(recipe_full_path):
recipe_full_path = str(
files(sltxpkg.data.recipes).joinpath(recipe_path))
if not os.path.isfile(recipe_full_path):
print_idx(
self.idx,
"Recipe " + recipe_full_path + " was not found. Exiting.")
exit(1)
print_idx(self.idx, "Loading recipe: " + recipe_full_path)
y_conf = su.load_yaml(recipe_full_path)
self.settings = {**self.settings, **y_conf}
if sg.args.quiet:
self.quiet = self.settings['quiet']
self.__process_tools()
self.__sanitize_extra_args() # we need them as a single string
def grab_stuff(idx: str, dep_name: str, target_dir: str, data: dict,
target: str):
extra_files, extra_dirs = extend_grab_from_local(idx, target_dir, data)
print_idx(idx, " > Grabbing dependencies for " + dep_name)
print_idx(idx,
" - Grabby-Grab-Grab files from \"" + target_dir + "\"...")
got_files = grab_from(idx, target_dir, data, target, 'grab-files',
f_grab_files, extra_files)
print_idx(idx, " - Grabby-Grab-Grab dirs from \"" + target_dir + "\"...")
got_dirs = grab_from(idx, target_dir, data, target, 'grab-dirs',
f_grab_dirs, extra_dirs)
if not got_files and not got_dirs:
print_idx(idx, " ! No grabs performed!")
write_to_log("No grabs performed for: " + dep_name)
def use_driver(idx: str, data: dict, dep_name: str, driver: str, target: str):
# default no arguments
if "args" not in data:
data["args"] = ""
driver_data = sg.configuration[C_DRIVERS][driver]
command = driver_data["command"].format(**data,
**sg.configuration,
dep_name=dep_name)
driver_target_dir = get_target_dir(data, dep_name, driver)
if driver_data["needs-delete"] and os.path.isdir(driver_target_dir):
print_idx(
idx, " - Target folder " + driver_target_dir +
"exists. Will be deleted as the driver needs this")
shutil.rmtree(driver_target_dir)
if driver_data["needs-create"] and not os.path.isdir(driver_target_dir):
print_idx(idx,
" - Target folder " + driver_target_dir + " needs creation")
os.makedirs(driver_target_dir)
print_idx(idx, " > Executing: " + command)
with Popen(command, stdout=PIPE, stderr=PIPE, shell=True) as feedback:
return_code = feedback.wait()
write_proc_to_log(idx, feedback.stdout, False)
if return_code != 0:
print_idx(idx, " - Error-Log of Driver:")
write_proc_to_log(idx, feedback.stderr, return_code != 0)
if sg.configuration[C_RECURSIVE]:
recursive_dependencies(idx, driver_target_dir, data, dep_name, target)
if return_code != 0:
print_idx(idx,
" ! Driver failed with code" + str(feedback) + "exiting.")
sys.exit(return_code)
grab_stuff(idx, dep_name, driver_target_dir, data, target)
def __runcmds(self, cmds: List[str]):
for cmd in cmds:
cmd = self.__f(cmd) # expand
if sg.args.verbose:
print_idx(self.idx, " - " + cmd)
os.system(cmd)
def run(self):
"""Executes the configured Recipe
"""
print_idx(self.idx, "Processing file: " + self.file, pre='\n')
sc.assure_dirs() # Ensure Working diSr and texmf home
sc.assure_dir('file cache', self.__f("{out_dir}"), True)
print_idx(self.idx,
self.__f("> Running recipe \"{name}\" by \"{author}\"."))
self.__runhooks('pre')
print_idx(
self.idx, "> Running the compile commands (" +
str(len(self.settings['run'])) + ")")
for i, cmd in enumerate(self.settings['run']):
cmd = self.__f(cmd) # expand
if sg.args.verbose:
print_idx(self.idx, " - " + cmd)
fback = os.system(cmd)
if fback != 0:
print_idx(
self.idx,
"\033[31m ! The command failed. Initiating critical abort...\033[m"
)
self.__critical_abort(fback)
self.__runhooks('in')
self.__save_files(os.getcwd())
self.__runhooks('post')
if sg.configuration[sg.C_CLEANUP]:
print_idx(self.idx, "> Cleaning up (configured by configuration)")
self.__runcmds(self.settings['cleanup_cmds'])
def __runhooks(self, hookid: str):
if sg.args.verbose:
print_idx(self.idx, "> Hooks for \"" + hookid + "\"")
self.__runcmds(self.settings['hooks'][hookid])
def install_dependency(name: str, idx: str, data: dict, target: str):
print_idx(idx, 'Loading "' + name + '"')
if "url" not in data:
print_idx(idx, " ! The dependency did not have an url-tag attached")
exit(1)
url = data["url"]
print_idx(idx, ' - Loading from: "' + url + '"')
if "driver" not in data:
if not sg.configuration[C_AUTODETECT_DRIVERS]:
print_idx(idx, " ! No driver given and auto-detection disabled!")
else:
data["driver"] = detect_driver(idx, url)
driver = data["driver"]
print_idx(idx, " - Using driver: \"" + driver + "\"")
if name in loaded:
print_idx(
idx, " > Skipping retrieval " + name +
" as it was already loaded by another dep.")
grab_stuff(idx, name, get_target_dir(data, name, driver), data, target)
return
loaded.append(name)
if driver not in sg.configuration[C_DRIVERS]:
print_idx(
idx, ' ! The selected driver is unknown. Loaded:' +
sg.configuration[C_DRIVERS])
sys.exit(2)
use_driver(idx, data, name, driver, target)
