def end_make_logging(makelog='@DEFAULTVALUE@'): """End "makelog" log file with time stamp. - To end make.py by putting the time stamp in the default '../output/make.log', use: `end_make_logging()` - If the log file is different, we have to specify it: `end_make_logging('logfile.txt')` """ if makelog == '@DEFAULTVALUE@': makelog = metadata.settings['makelog_file'] if not (metadata.makelog_started and os.path.isfile(makelog)): raise CritError(messages.crit_error_nomakelog % makelog) print("\nEnd log file: ", makelog) makelog = re.sub('\\\\', '/', makelog) try: LOGFILE = open(makelog, 'a') except Exception as errmsg: raise CritError((messages.crit_error_log % makelog) + '\n' + str(errmsg)) time_end = datetime.datetime.now().replace(microsecond=0) print(messages.note_makelogend, time_end, file=LOGFILE) LOGFILE.close()
def _insert_tables_lyx(template, tables): """.. Fill tables for LyX template. Parameters ---------- template : str Path of LyX template to fill. tables : dict Dictionary ``{tag: values}`` of tables. Returns ------- template : str Filled LyX template. """ with open(template, 'r') as f: doc = f.readlines() is_table = False for i in range(len(doc)): if re.match('name "tab:', doc[i]): tag = doc[i].replace('name "tab:','').rstrip('"\n').lower() try: values = tables[tag] entry_count = 0 is_table = True except KeyError: pass while is_table: try: if re.match('.*###', doc[i]): doc[i] = _insert_value(doc[i], values[entry_count], 'no change') entry_count += 1 break elif re.match('.*#[0-9]+#', doc[i]): doc[i] = _insert_value(doc[i], values[entry_count], 'round') entry_count += 1 break elif re.match('.*#[0-9]+,#', doc[i]): doc[i] = _insert_value(doc[i], values[entry_count], 'comma + round') entry_count += 1 break elif re.match('</lyxtabular>', doc[i]): is_table = False if entry_count != len(values): raise_from(CritError(messages.crit_error_too_many_values % tag), None) else: break except IndexError: raise_from(CritError(messages.crit_error_not_enough_values % tag), None) doc = '\n'.join(doc) return(doc)
def execute_run(self, command): print('\n') current_directory = os.getcwd() if self.changedir: os.chdir(self.program_path) if not self.log: tempname = current_directory + '/make-templog.txt' else: tempname = os.path.abspath(self.log) TEMPFILE = open(tempname, 'w') if self.makelog: if not (metadata.makelog_started and os.path.isfile(self.makelog)): raise CritError(messages.crit_error_nomakelog % self.makelog) # Open main log file try: LOGFILE = open(self.makelog, 'a') except Exception as errmsg: print(errmsg) raise CritError(messages.crit_error_log % self.makelog) try: # Execute command and print content to LOGFILE print('Executing: ', command) print('\n\nExecute: ', command, file=LOGFILE) subprocess.check_call(command, shell = True, stdout = TEMPFILE, stderr = TEMPFILE) TEMPFILE.close() LOGFILE.write(open(tempname, 'rU').read()) LOGFILE.close() except Exception as errmsg: # If fails then print errors to LOGFILE TEMPFILE.close() LOGFILE.write(open(tempname, 'rU').read()) print(messages.crit_error_bad_command % command, '\n', str(errmsg)) print(messages.crit_error_bad_command % command, '\n', str(errmsg), file=LOGFILE) LOGFILE.close() else: try: # Execute command print('Executing: ', command) subprocess.check_call(command, shell = True, stdout = TEMPFILE, stderr = TEMPFILE) TEMPFILE.close() except Exception as errmsg: # If fails then print errors TEMPFILE.close() print(messages.crit_error_bad_command % command, '\n', str(errmsg)) print(messages.crit_error_bad_command % command, '\n', str(errmsg), file=TEMPFILE) if not self.log: os.remove(tempname) if self.changedir: os.chdir(current_directory)
def check_program(self): """Check program exists and has correct extension given application. Returns ------- None """ if not os.path.isfile(self.program): raise CritError(messages.crit_error_no_file % self.program) if self.program_ext not in metadata.extensions[self.application]: extensions = format_list(metadata.extensions[self.application]) raise CritError(messages.crit_error_extension % (self.program, extensions))
def _insert_value(line, value, type, null): """.. Insert value into line. Parameters ---------- line : str Line of document to insert value. value : str Value to insert. type : str Formatting for value. Returns ------- line : str Line of document with inserted value. """ if (type == 'no change'): line = re.sub('\\\\?#\\\\?#\\\\?#', value, line) elif (type == 'round'): if value == null: line = re.sub('(.*?)\\\\?#[0-9]+\\\\?#', r'\g<1>' + value, line) else: try: value = float(value) except: raise_from(CritError(messages.crit_error_not_float % value), None) digits = re.findall('\\\\?#([0-9]+)\\\\?#', line)[0] rounded_value = format(value, '.%sf' % digits) line = re.sub('(.*?)\\\\?#[0-9]+\\\\?#', r'\g<1>' + rounded_value, line) elif (type == 'comma + round'): if value == null: line = re.sub('(.*?)\\\\?#[0-9]+,\\\\?#', r'\g<1>' + value, line) else: try: value = float(value) except: raise_from(CritError(messages.crit_error_not_float % value), None) digits = re.findall('\\\\?#([0-9]+),\\\\?#', line)[0] rounded_value = format(value, ',.%sf' % digits) line = re.sub('(.*?)\\\\?#[0-9]+,\\\\?#', r'\g<1>' + rounded_value, line) return (line)
def _parse_git_attributes(attributes): """.. Get git lfs patterns from git attributes. Get git lfs patterns from file ``attributes``. Parameters ---------- attributes : str Path of git attributes file. Returns ------- lfs_list: list List of patterns to determine files tracked by git lfs. """ try: with open(attributes) as f: attributes_list = f.readlines() lfs_regex = 'filter=lfs( )+diff=lfs( )+merge=lfs( )+-text' lfs_list = [l for l in attributes_list if re.search(lfs_regex, l)] lfs_list = [l.split()[0] for l in lfs_list] return (lfs_list) except IOError: raise_from(CritError(messages.crit_error_no_attributes), None)
def get_path(paths_dict, key, throw_error=True): """Get path for key. Parameters ---------- path_dict : dict Dictionary of paths. key : str Path to get from dictionary. throw_error : bool Return error instead of `None`. Defaults to `True`. Returns ------- path : str Path requested. """ try: path = paths_dict[key] except KeyError: if throw_error: raise_from(CritError(messages.crit_error_no_key % (key, key)), None) else: path = None return (path)
def write_to_makelog(paths, message): """.. Write to make log. Appends string ``message`` to file ``makelog``. Parameters ---------- paths : dict Dictionary of paths. Dictionary should contain values for all keys listed below. message : str Message to append. Path Keys --------- makelog : str Path of makelog. Returns ------- None """ makelog = get_path(paths, 'makelog') if makelog: makelog = norm_path(makelog) if not (metadata.makelog_started and os.path.isfile(makelog)): raise_from(CritError(messages.crit_error_no_makelog % makelog), None) with io.open(makelog, 'a', encoding = 'utf8', errors = 'ignore') as MAKELOG: print(message, file = MAKELOG)
def get_path(paths_dict, key, throw_error=True): """Get path for key. Parameters ---------- path_dict : dict Dictionary of paths. key : str Path to get from dictionary. throw_error : bool Return error instead of ``None``. Defaults to ``True``. Returns ------- path : str Path requested. """ try: path = paths_dict[key] if isinstance(path, string_types): path = norm_path(path) elif isinstance(path, list): path = [norm_path(p) for p in path] except KeyError: if throw_error: raise_from(CritError(messages.crit_error_no_key % (key, key)), None) else: path = None return (path)
def error_check(self, prog): if (self.osname != 'posix') & (self.osname != 'nt'): raise CritError(messages.crit_error_unknown_system % self.osname) ext = metadata.extensions[prog] if self.program_ext == '': self.program_ext = ext if self.program_ext: self.program = self.program_name + self.program_ext self.program_full = os.path.join(self.program_path, self.program) if not os.path.isfile(self.program_full): raise CritError(messages.crit_error_no_file % self.program_full) if self.program_ext != ext: raise CritError(messages.crit_error_extension % self.program_full)
def add_error_to_log(makelog): if not makelog: return if not (metadata.makelog_started and os.path.isfile(makelog)): raise CritError(messages.crit_error_nomakelog % makelog) LOGFILE = open(makelog, 'a') print_error(LOGFILE) LOGFILE.close()
def move_posix(self, movetype): """Create symlinks/copies using POSIX shell command specified in metadata. Parameters ---------- movetype : str Type of file movement. Takes either `copy` or `symlink`. Returns ------- None """ for source, destination in self.move_list: if movetype == 'copy': command = metadata.commands[self.osname]['makecopy'] % ( source, destination) elif movetype == 'symlink': command = metadata.commands[self.osname]['makelink'] % ( source, destination) process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) stdout, stderr = process.communicate() if process.returncode != 0: error_message = messages.crit_error_move_command % command error_message = error_message + format_traceback(stderr) raise CritError(error_message)
def move_program_output(self, program_output, log_file=''): """Move program outputs. Notes ----- Certain applications create program outputs that need to be moved to appropriate logging files. Parameters ---------- program_output : str Path of program output. log_file : str, optional Path of log file. Log file is only written if specified. Defaults to ``''`` (i.e., not written). """ program_output = norm_path(program_output) try: with io.open(program_output, 'r', encoding='utf-8', errors='ignore') as f: out = f.read() except: error_message = messages.crit_error_no_program_output % ( program_output, self.program) error_message = error_message + format_traceback() raise_from(CritError(error_message), None) if self.makelog: if not (metadata.makelog_started and os.path.isfile(self.makelog)): raise CritError(messages.crit_error_no_makelog % self.makelog) with io.open(self.makelog, 'a', encoding='utf-8', errors='ignore') as f: print(out, file=f) if log_file: if program_output != log_file: shutil.copy2(program_output, log_file) os.remove(program_output) else: os.remove(program_output) return (out)
def option_overlap_error_check(self, kwargs): prog = [prog for prog, ext in metadata.extensions.iteritems() if ext == self.program_ext][0] option_overlaps = metadata.option_overlaps.get(prog) if not option_overlaps: return for opt in option_overlaps: if self.option_dict.has_key(option_overlaps[opt]) and kwargs.has_key(opt): raise CritError(messages.crit_error_option_overlap % (opt, option_overlaps[opt]))
def _insert_tables_latex(template, tables): """.. Fill tables for LaTeX template. Parameters ---------- template : str Path of LaTeX template to fill. tables : dict Dictionary ``{tag: values}`` of tables. Returns ------- template : str Filled LaTeX template. """ with open(template, 'r') as f: doc = f.readlines() is_table = False for i in range(len(doc)): if re.search('label\{tab:', doc[i]): tag = doc[i].split(':')[1].rstrip('}\n').strip('"').lower() try: values = tables[tag] entry_count = 0 is_table = True except KeyError: pass while is_table: line_col = doc[i].split("&") for j in range(len(line_col)): if re.search('.*\\\\#\\\\#\\\\#', line_col[j]): line_col[j] = _insert_value(line_col[j], values[entry_count], 'no change') entry_count += 1 elif re.search('.*\\\\#[0-9]+\\\\#', line_col[j]): line_col[j] = _insert_value(line_col[j], values[entry_count], 'round') entry_count += 1 elif re.search('.*\\\\#[0-9]+,\\\\#', line_col[j]): line_col[j] = _insert_value(line_col[j], values[entry_count], 'comma + round') entry_count += 1 doc[i] = "&".join(line_col) if re.search('end\{tabular\}', doc[i], flags = re.IGNORECASE): is_table = False if entry_count != len(values): raise_from(CritError(messages.crit_error_too_many_values % tag), None) else: break doc = '\n'.join(doc) return(doc)
def get_modified_sources(paths, source_map, depth=float('inf')): """.. Get source files considered changed by git. Checks the modification status for all sources contained in list ``source_map`` (returned by `sourcing functions`_). Produces warning if sources have been modified according to git. When walking through sources, float ``depth`` determines level of depth to walk. Warning messages are appended to file ``makelog``. Parameters ---------- paths : dict Dictionary of paths. Dictionary should contain values for all keys listed below. source_map : list Mapping of sources (returned from `sourcing functions`_). depth : float, optional Level of depth when walking through source directories. Defaults to infinite. Path Keys --------- makelog : str Path of makelog. Returns ------- overlap : list List of source files considered changed by git. Notes ----- """ try: source_list = [source for source, destination in source_map] source_list = [glob_recursive(source, depth) for source in source_list] source_files = [f for source in source_list for f in source] source_files = set(source_files) try: repo = git.Repo('.', search_parent_directories=True) except: raise_from(CritError(messages.crit_error_no_repo), None) modified = _get_git_status(repo) overlap = [l for l in source_files if l in modified] if overlap: if len(overlap) > 100: overlap = overlap[0:100] overlap = overlap + [ "and more (file list truncated due to length)" ] message = messages.warning_modified_files % '\n'.join(overlap) write_to_makelog(paths, message) print(colored(message, metadata.color_failure)) except: error_message = 'Error with `get_modified_sources`. Traceback can be found below.' error_message = format_message(error_message) write_to_makelog(paths, error_message + '\n\n' + traceback.format_exc()) raise_from(ColoredError(error_message, traceback.format_exc()), None)
def move_log(self, default_log): if self.makelog: if not (metadata.makelog_started and os.path.isfile(self.makelog)): raise CritError(messages.crit_error_nomakelog % self.makelog) if os.path.abspath(default_log) != os.path.abspath(self.log): # Append default_log to main log LOGFILE = open(self.makelog, 'a') try: LOGFILE.write(open(default_log, 'rU').read()) except Exception as errmsg: print(errmsg) raise CritError(messages.crit_error_no_file % default_log) LOGFILE.close() # Save default_log as self.log if self.log: shutil.copy2(default_log, self.log) os.remove(default_log)
def del_log(*args, **kwargs): """Delete log files This function deletes each of the log files listed in "*arg" list. Errors are printed to `makelog` log file. - After we append the various log files to the common one using add_log, we may want to delete the extra ones to clean up the output folder. Here is the command: `del_log('../output/analysis.log', '../output/another_log.log') - If variable 'makelog' is not defined in kwargs, errors from del_log will be printed to the default makelog (usually '../output/make.log'). If we want to specify a different log to which we print the errors: `del_log('../output/analysis.log', '../output/another_log.log', makelog = '../different.log')` """ if 'makelog' in kwargs.keys(): makelog = kwargs['makelog'] else: makelog = metadata.settings['makelog_file'] if not (metadata.makelog_started and os.path.isfile(makelog)): raise CritError(messages.crit_error_nomakelog % makelog) print("\nDelete log file(s)") makelog = re.sub('\\\\', '/', makelog) try: LOGFILE = open(makelog, 'a') except Exception as errmsg: raise CritError((messages.crit_error_log % makelog) + '\n' + str(errmsg)) try: for log in args: log = re.sub('\\\\', '/', log) if os.path.isfile(log): os.remove(log) else: print(messages.note_nofile % log, file=LOGFILE) except: print_error(LOGFILE) LOGFILE.close()
def check_paths(self): """Check sources and destination exist and have same number of wildcards. Returns ------- None """ if re.findall('\*', self.source) != re.findall('\*', self.destination): raise SyntaxError(messages.syn_error_wildcard % (self.raw_line, self.file)) if re.search('\*', self.source): if not glob.glob(self.source): raise CritError(messages.crit_error_no_path_wildcard % self.source) else: if not os.path.exists(self.source): raise CritError(messages.crit_error_no_path % self.source)
def check_os(self): """Check OS is either POSIX or NT. Returns ------- None """ if self.osname not in ['posix', 'nt']: raise CritError(messages.crit_error_unknown_system % self.osname)
def run_module(root, module, build_script='make.py', osname=None): """.. Run module. Runs script `build_script` in module directory `module` relative to root of repository `root`. Parameters ---------- root : str Directory of root. module: str Name of module. build_script : str Name of build script. Defaults to ``make.py``. osname : str, optional Name of OS. Used to determine syntax of system command. Defaults to ``os.name``. Returns ------- None Example ------- The following code runs the script ``root/module/make.py``. .. code-block:: python run_module(root = 'root', module = 'module') """ osname = osname if osname else os.name # https://github.com/sphinx-doc/sphinx/issues/759 try: module_dir = os.path.join(root, module) os.chdir(module_dir) build_script = norm_path(build_script) if not os.path.isfile(build_script): raise CritError(messages.crit_error_no_file % build_script) message = 'Running module `%s`' % module message = format_message(message) message = colored(message, attrs=['bold']) print('\n' + message) status = os.system( '%s %s' % (metadata.default_executables[osname]['python'], build_script)) if status != 0: raise ProgramError() except ProgramError: sys.exit() except: error_message = 'Error with `run_module`. Traceback can be found below.' error_message = format_message(error_message) raise_from(ColoredError(error_message, traceback.format_exc()), None)
def add_log(*args, **kwargs): """Add log files in "*arg" list to "makelog" log file. - If there are log files created inside scripts (for example from a Perl or Python script), and we want to append their content to the end of the default log file (usually '../output/make.log'), we would use: `add_log('../output/analysis.log', '../output/another_log.log')` - The number of log arguments can vary. If the log files don't actually exist, errors will be printed to the common log file. - If we want to append content of log files to a different log file than the default, we would use: `add_log('../output/analysis.log', '../output/another_log.log', makelog = '../different.log')` """ if 'makelog' in kwargs.keys(): makelog = kwargs['makelog'] else: makelog = metadata.settings['makelog_file'] if not (metadata.makelog_started and os.path.isfile(makelog)): raise CritError(messages.crit_error_nomakelog % makelog) print("\nAdd log file(s) to: ", makelog) makelog = re.sub('\\\\', '/', makelog) try: LOGFILE = open(makelog, 'a') except Exception as errmsg: raise CritError((messages.crit_error_log % makelog) + '\n' + str(errmsg)) try: for log in args: log = re.sub('\\\\', '/', log) if not os.path.isfile(log): print(messages.note_nofile % log, file=LOGFILE) else: LOGFILE.write(open(log, 'rU').read()) except: print_error(LOGFILE) LOGFILE.close()
def end_logging(LOGFILE, makelog, logtype): time_end = datetime.datetime.now().replace(microsecond=0) print(messages.note_logend % logtype, time_end, file=LOGFILE) LOGFILE.close() if not makelog: return if not (metadata.makelog_started and os.path.isfile(makelog)): raise CritError(messages.crit_error_nomakelog % makelog) MAKE_LOGFILE = open(makelog, 'a') MAKE_LOGFILE.write(open(LOGFILE.name, 'rU').read()) MAKE_LOGFILE.close() os.remove(LOGFILE.name)
def _parse_content(file, null): """.. Parse content from input.""" with io.open(file, 'r', encoding='utf-8') as f: content = f.readlines() try: tag = _parse_tag(content[0]) except: raise_from(CritError(messages.crit_error_no_tag % file), None) data = _parse_data(content[1:], null) return (tag, data)
def _check_os(osname=os.name): """Check OS is either POSIX or NT. Parameters ---------- osname : str, optional Name of OS. Defaults to ``os.name``. Returns ------- None """ if osname not in ['posix', 'nt']: raise CritError(messages.crit_error_unknown_system % osname)
def end_makelog(paths): """.. End make log. Appends to file ``makelog``, recording end time. Note ---- We allow for writing to a make log even after the make log has ended. We do not recommend this for best practice. Parameters ---------- paths : dict Dictionary of paths. Dictionary should contain values for all keys listed below. Path Keys --------- makelog : str Path of makelog. Returns ------- None """ try: makelog = get_path(paths, 'makelog') if makelog: makelog = norm_path(makelog) message = 'Ending makelog file at: `%s`' % makelog print(colored(message, metadata.color_success)) if not (metadata.makelog_started and os.path.isfile(makelog)): raise_from(CritError(messages.crit_error_no_makelog % makelog), None) with open(makelog, 'a', encoding='utf8') as MAKELOG: time_end = str(datetime.datetime.now().replace(microsecond=0)) working_dir = os.getcwd() print(messages.note_dash_line, file=MAKELOG) print(messages.note_makelog_end + time_end, file=MAKELOG) print(messages.note_working_directory + working_dir, file=MAKELOG) print(messages.note_dash_line, file=MAKELOG) except: error_message = 'Error with `end_makelog`. Traceback can be found below.' error_message = format_message(error_message) raise_from(ColoredError(error_message, traceback.format_exc()), None)
def write_log(self): """Write logs for shell command. Returns ------- None """ if self.makelog: if not (metadata.makelog_started and os.path.isfile(self.makelog)): raise CritError(messages.crit_error_no_makelog % self.makelog) with open(self.makelog, 'a') as f: print(self.output, file = f) if self.log: with open(self.log, 'w') as f: f.write(self.output)
def input_to_array(filename): # Import file try: FILENAME = open(filename, 'rU') except: raise CritError(messages.crit_error_file % filename) # Delete header filearray = [] for line in FILENAME: if (not re.match('rev', line) and not re.match('linkpath', line) and not re.match('\s*\#', line) and not re.match('\s*$', line) and not re.match('url', line)): filearray.append(line.rstrip('\n')) FILENAME.close() return filearray
def parse_file_list(self): """Parse wildcards in list of files. Returns ------- None """ self.file_list = convert_to_list(file_list, 'file') file_list_parsed = [ f for file in self.file_list for f in glob.glob(file) ] if file_list_parsed: self.file_list = file_list_parsed else: error_list = [str(f) for f in self.file_list] raise CritError(messages.crit_error_no_files % error_list)
def execute_command(self, command): """Execute shell command. Parameters ---------- command : str Shell command to execute. Returns ------- exit : tuple Tuple (exit code, error message) for shell command. """ self.output = 'Executing command: `%s`' % command print(colored(self.output, metadata.color_in_process)) try: if not self.shell: command = command.split() process = subprocess_fix.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=self.shell, universal_newlines=True) process.wait() stdout, stderr = process.communicate() exit = (process.returncode, stderr) if stdout: self.output += '\n' + decode(stdout) if stderr: self.output += '\n' + decode(stderr) pass return (exit) except: error_message = messages.crit_error_bad_command % command error_message = error_message + format_traceback() raise_from(CritError(error_message), None)