def end_make_logging(makelog='@DEFAULTVALUE@'):
"""End "makelog" log file with time stamp.
- To end make.py by putting the time stamp in the default
'../output/make.log', use: `end_make_logging()`
- If the log file is different, we have to specify it:
`end_make_logging('logfile.txt')`
"""
if makelog == '@DEFAULTVALUE@':
makelog = metadata.settings['makelog_file']
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise CritError(messages.crit_error_nomakelog % makelog)
print("\nEnd log file: ", makelog)
makelog = re.sub('\\\\', '/', makelog)
try:
LOGFILE = open(makelog, 'a')
except Exception as errmsg:
raise CritError((messages.crit_error_log % makelog) + '\n' +
str(errmsg))
time_end = datetime.datetime.now().replace(microsecond=0)
print(messages.note_makelogend, time_end, file=LOGFILE)
LOGFILE.close()
def _insert_tables_lyx(template, tables):
""".. Fill tables for LyX template.
Parameters
----------
template : str
Path of LyX template to fill.
tables : dict
Dictionary ``{tag: values}`` of tables.
Returns
-------
template : str
Filled LyX template.
"""
with open(template, 'r') as f:
doc = f.readlines()
is_table = False
for i in range(len(doc)):
if re.match('name "tab:', doc[i]):
tag = doc[i].replace('name "tab:','').rstrip('"\n').lower()
try:
values = tables[tag]
entry_count = 0
is_table = True
except KeyError:
pass
while is_table:
try:
if re.match('.*###', doc[i]):
doc[i] = _insert_value(doc[i], values[entry_count], 'no change')
entry_count += 1
break
elif re.match('.*#[0-9]+#', doc[i]):
doc[i] = _insert_value(doc[i], values[entry_count], 'round')
entry_count += 1
break
elif re.match('.*#[0-9]+,#', doc[i]):
doc[i] = _insert_value(doc[i], values[entry_count], 'comma + round')
entry_count += 1
break
elif re.match('</lyxtabular>', doc[i]):
is_table = False
if entry_count != len(values):
raise_from(CritError(messages.crit_error_too_many_values % tag), None)
else:
break
except IndexError:
raise_from(CritError(messages.crit_error_not_enough_values % tag), None)
doc = '\n'.join(doc)
return(doc)
def execute_run(self, command):
print('\n')
current_directory = os.getcwd()
if self.changedir:
os.chdir(self.program_path)
if not self.log:
tempname = current_directory + '/make-templog.txt'
else:
tempname = os.path.abspath(self.log)
TEMPFILE = open(tempname, 'w')
if self.makelog:
if not (metadata.makelog_started and os.path.isfile(self.makelog)):
raise CritError(messages.crit_error_nomakelog % self.makelog)
# Open main log file
try:
LOGFILE = open(self.makelog, 'a')
except Exception as errmsg:
print(errmsg)
raise CritError(messages.crit_error_log % self.makelog)
try:
# Execute command and print content to LOGFILE
print('Executing: ', command)
print('\n\nExecute: ', command, file=LOGFILE)
subprocess.check_call(command, shell = True, stdout = TEMPFILE, stderr = TEMPFILE)
TEMPFILE.close()
LOGFILE.write(open(tempname, 'rU').read())
LOGFILE.close()
except Exception as errmsg:
# If fails then print errors to LOGFILE
TEMPFILE.close()
LOGFILE.write(open(tempname, 'rU').read())
print(messages.crit_error_bad_command % command, '\n', str(errmsg))
print(messages.crit_error_bad_command % command, '\n', str(errmsg), file=LOGFILE)
LOGFILE.close()
else:
try:
# Execute command
print('Executing: ', command)
subprocess.check_call(command, shell = True, stdout = TEMPFILE, stderr = TEMPFILE)
TEMPFILE.close()
except Exception as errmsg:
# If fails then print errors
TEMPFILE.close()
print(messages.crit_error_bad_command % command, '\n', str(errmsg))
print(messages.crit_error_bad_command % command, '\n', str(errmsg), file=TEMPFILE)
if not self.log:
os.remove(tempname)
if self.changedir:
os.chdir(current_directory)
def check_program(self):
"""Check program exists and has correct extension given application.
Returns
-------
None
"""
if not os.path.isfile(self.program):
raise CritError(messages.crit_error_no_file % self.program)
if self.program_ext not in metadata.extensions[self.application]:
extensions = format_list(metadata.extensions[self.application])
raise CritError(messages.crit_error_extension % (self.program, extensions))
def _insert_value(line, value, type, null):
""".. Insert value into line.
Parameters
----------
line : str
Line of document to insert value.
value : str
Value to insert.
type : str
Formatting for value.
Returns
-------
line : str
Line of document with inserted value.
"""
if (type == 'no change'):
line = re.sub('\\\\?#\\\\?#\\\\?#', value, line)
elif (type == 'round'):
if value == null:
line = re.sub('(.*?)\\\\?#[0-9]+\\\\?#', r'\g<1>' + value, line)
else:
try:
value = float(value)
except:
raise_from(CritError(messages.crit_error_not_float % value),
None)
digits = re.findall('\\\\?#([0-9]+)\\\\?#', line)[0]
rounded_value = format(value, '.%sf' % digits)
line = re.sub('(.*?)\\\\?#[0-9]+\\\\?#', r'\g<1>' + rounded_value,
line)
elif (type == 'comma + round'):
if value == null:
line = re.sub('(.*?)\\\\?#[0-9]+,\\\\?#', r'\g<1>' + value, line)
else:
try:
value = float(value)
except:
raise_from(CritError(messages.crit_error_not_float % value),
None)
digits = re.findall('\\\\?#([0-9]+),\\\\?#', line)[0]
rounded_value = format(value, ',.%sf' % digits)
line = re.sub('(.*?)\\\\?#[0-9]+,\\\\?#', r'\g<1>' + rounded_value,
line)
return (line)
def _parse_git_attributes(attributes):
""".. Get git lfs patterns from git attributes.
Get git lfs patterns from file ``attributes``.
Parameters
----------
attributes : str
Path of git attributes file.
Returns
-------
lfs_list: list
List of patterns to determine files tracked by git lfs.
"""
try:
with open(attributes) as f:
attributes_list = f.readlines()
lfs_regex = 'filter=lfs( )+diff=lfs( )+merge=lfs( )+-text'
lfs_list = [l for l in attributes_list if re.search(lfs_regex, l)]
lfs_list = [l.split()[0] for l in lfs_list]
return (lfs_list)
except IOError:
raise_from(CritError(messages.crit_error_no_attributes), None)
def get_path(paths_dict, key, throw_error=True):
"""Get path for key.
Parameters
----------
path_dict : dict
Dictionary of paths.
key : str
Path to get from dictionary.
throw_error : bool
Return error instead of `None`. Defaults to `True`.
Returns
-------
path : str
Path requested.
"""
try:
path = paths_dict[key]
except KeyError:
if throw_error:
raise_from(CritError(messages.crit_error_no_key % (key, key)),
None)
else:
path = None
return (path)
def write_to_makelog(paths, message):
""".. Write to make log.
Appends string ``message`` to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
message : str
Message to append.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
makelog = get_path(paths, 'makelog')
if makelog:
makelog = norm_path(makelog)
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise_from(CritError(messages.crit_error_no_makelog % makelog), None)
with io.open(makelog, 'a', encoding = 'utf8', errors = 'ignore') as MAKELOG:
print(message, file = MAKELOG)
def get_path(paths_dict, key, throw_error=True):
"""Get path for key.
Parameters
----------
path_dict : dict
Dictionary of paths.
key : str
Path to get from dictionary.
throw_error : bool
Return error instead of ``None``. Defaults to ``True``.
Returns
-------
path : str
Path requested.
"""
try:
path = paths_dict[key]
if isinstance(path, string_types):
path = norm_path(path)
elif isinstance(path, list):
path = [norm_path(p) for p in path]
except KeyError:
if throw_error:
raise_from(CritError(messages.crit_error_no_key % (key, key)),
None)
else:
path = None
return (path)
def error_check(self, prog):
if (self.osname != 'posix') & (self.osname != 'nt'):
raise CritError(messages.crit_error_unknown_system % self.osname)
ext = metadata.extensions[prog]
if self.program_ext == '':
self.program_ext = ext
if self.program_ext:
self.program = self.program_name + self.program_ext
self.program_full = os.path.join(self.program_path, self.program)
if not os.path.isfile(self.program_full):
raise CritError(messages.crit_error_no_file % self.program_full)
if self.program_ext != ext:
raise CritError(messages.crit_error_extension % self.program_full)
def add_error_to_log(makelog):
if not makelog: return
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise CritError(messages.crit_error_nomakelog % makelog)
LOGFILE = open(makelog, 'a')
print_error(LOGFILE)
LOGFILE.close()
def move_posix(self, movetype):
"""Create symlinks/copies using POSIX shell command specified in metadata.
Parameters
----------
movetype : str
Type of file movement. Takes either `copy` or `symlink`.
Returns
-------
None
"""
for source, destination in self.move_list:
if movetype == 'copy':
command = metadata.commands[self.osname]['makecopy'] % (
source, destination)
elif movetype == 'symlink':
command = metadata.commands[self.osname]['makelink'] % (
source, destination)
process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
stdout, stderr = process.communicate()
if process.returncode != 0:
error_message = messages.crit_error_move_command % command
error_message = error_message + format_traceback(stderr)
raise CritError(error_message)
def move_program_output(self, program_output, log_file=''):
"""Move program outputs.
Notes
-----
Certain applications create program outputs that need to be moved to
appropriate logging files.
Parameters
----------
program_output : str
Path of program output.
log_file : str, optional
Path of log file. Log file is only written if specified.
Defaults to ``''`` (i.e., not written).
"""
program_output = norm_path(program_output)
try:
with io.open(program_output,
'r',
encoding='utf-8',
errors='ignore') as f:
out = f.read()
except:
error_message = messages.crit_error_no_program_output % (
program_output, self.program)
error_message = error_message + format_traceback()
raise_from(CritError(error_message), None)
if self.makelog:
if not (metadata.makelog_started and os.path.isfile(self.makelog)):
raise CritError(messages.crit_error_no_makelog % self.makelog)
with io.open(self.makelog, 'a', encoding='utf-8',
errors='ignore') as f:
print(out, file=f)
if log_file:
if program_output != log_file:
shutil.copy2(program_output, log_file)
os.remove(program_output)
else:
os.remove(program_output)
return (out)
def option_overlap_error_check(self, kwargs):
prog = [prog for prog, ext in metadata.extensions.iteritems() if ext == self.program_ext][0]
option_overlaps = metadata.option_overlaps.get(prog)
if not option_overlaps: return
for opt in option_overlaps:
if self.option_dict.has_key(option_overlaps[opt]) and kwargs.has_key(opt):
raise CritError(messages.crit_error_option_overlap
% (opt, option_overlaps[opt]))
def _insert_tables_latex(template, tables):
""".. Fill tables for LaTeX template.
Parameters
----------
template : str
Path of LaTeX template to fill.
tables : dict
Dictionary ``{tag: values}`` of tables.
Returns
-------
template : str
Filled LaTeX template.
"""
with open(template, 'r') as f:
doc = f.readlines()
is_table = False
for i in range(len(doc)):
if re.search('label\{tab:', doc[i]):
tag = doc[i].split(':')[1].rstrip('}\n').strip('"').lower()
try:
values = tables[tag]
entry_count = 0
is_table = True
except KeyError:
pass
while is_table:
line_col = doc[i].split("&")
for j in range(len(line_col)):
if re.search('.*\\\\#\\\\#\\\\#', line_col[j]):
line_col[j] = _insert_value(line_col[j], values[entry_count], 'no change')
entry_count += 1
elif re.search('.*\\\\#[0-9]+\\\\#', line_col[j]):
line_col[j] = _insert_value(line_col[j], values[entry_count], 'round')
entry_count += 1
elif re.search('.*\\\\#[0-9]+,\\\\#', line_col[j]):
line_col[j] = _insert_value(line_col[j], values[entry_count], 'comma + round')
entry_count += 1
doc[i] = "&".join(line_col)
if re.search('end\{tabular\}', doc[i], flags = re.IGNORECASE):
is_table = False
if entry_count != len(values):
raise_from(CritError(messages.crit_error_too_many_values % tag), None)
else:
break
doc = '\n'.join(doc)
return(doc)
def get_modified_sources(paths, source_map, depth=float('inf')):
""".. Get source files considered changed by git.
Checks the modification status for all sources contained in list ``source_map`` (returned by `sourcing functions`_). Produces warning if sources have been modified according to git. When walking through sources, float ``depth`` determines level of depth to walk. Warning messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
source_map : list
Mapping of sources (returned from `sourcing functions`_).
depth : float, optional
Level of depth when walking through source directories. Defaults to infinite.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
overlap : list
List of source files considered changed by git.
Notes
-----
"""
try:
source_list = [source for source, destination in source_map]
source_list = [glob_recursive(source, depth) for source in source_list]
source_files = [f for source in source_list for f in source]
source_files = set(source_files)
try:
repo = git.Repo('.', search_parent_directories=True)
except:
raise_from(CritError(messages.crit_error_no_repo), None)
modified = _get_git_status(repo)
overlap = [l for l in source_files if l in modified]
if overlap:
if len(overlap) > 100:
overlap = overlap[0:100]
overlap = overlap + [
"and more (file list truncated due to length)"
]
message = messages.warning_modified_files % '\n'.join(overlap)
write_to_makelog(paths, message)
print(colored(message, metadata.color_failure))
except:
error_message = 'Error with `get_modified_sources`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def move_log(self, default_log):
if self.makelog:
if not (metadata.makelog_started and os.path.isfile(self.makelog)):
raise CritError(messages.crit_error_nomakelog % self.makelog)
if os.path.abspath(default_log) != os.path.abspath(self.log):
# Append default_log to main log
LOGFILE = open(self.makelog, 'a')
try:
LOGFILE.write(open(default_log, 'rU').read())
except Exception as errmsg:
print(errmsg)
raise CritError(messages.crit_error_no_file % default_log)
LOGFILE.close()
# Save default_log as self.log
if self.log:
shutil.copy2(default_log, self.log)
os.remove(default_log)
def del_log(*args, **kwargs):
"""Delete log files
This function deletes each of the log files listed in "*arg" list.
Errors are printed to `makelog` log file.
- After we append the various log files to the common one using add_log, we may
want to delete the extra ones to clean up the output folder. Here is the command:
`del_log('../output/analysis.log', '../output/another_log.log')
- If variable 'makelog' is not defined in kwargs, errors from del_log will be printed
to the default makelog (usually '../output/make.log'). If we want to specify a
different log to which we print the errors:
`del_log('../output/analysis.log', '../output/another_log.log',
makelog = '../different.log')`
"""
if 'makelog' in kwargs.keys():
makelog = kwargs['makelog']
else:
makelog = metadata.settings['makelog_file']
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise CritError(messages.crit_error_nomakelog % makelog)
print("\nDelete log file(s)")
makelog = re.sub('\\\\', '/', makelog)
try:
LOGFILE = open(makelog, 'a')
except Exception as errmsg:
raise CritError((messages.crit_error_log % makelog) + '\n' +
str(errmsg))
try:
for log in args:
log = re.sub('\\\\', '/', log)
if os.path.isfile(log):
os.remove(log)
else:
print(messages.note_nofile % log, file=LOGFILE)
except:
print_error(LOGFILE)
LOGFILE.close()
def check_paths(self):
"""Check sources and destination exist and have same number of wildcards.
Returns
-------
None
"""
if re.findall('\*', self.source) != re.findall('\*', self.destination):
raise SyntaxError(messages.syn_error_wildcard %
(self.raw_line, self.file))
if re.search('\*', self.source):
if not glob.glob(self.source):
raise CritError(messages.crit_error_no_path_wildcard %
self.source)
else:
if not os.path.exists(self.source):
raise CritError(messages.crit_error_no_path % self.source)
def check_os(self):
"""Check OS is either POSIX or NT.
Returns
-------
None
"""
if self.osname not in ['posix', 'nt']:
raise CritError(messages.crit_error_unknown_system % self.osname)
def run_module(root, module, build_script='make.py', osname=None):
""".. Run module.
Runs script `build_script` in module directory `module` relative to root of repository `root`.
Parameters
----------
root : str
Directory of root.
module: str
Name of module.
build_script : str
Name of build script. Defaults to ``make.py``.
osname : str, optional
Name of OS. Used to determine syntax of system command. Defaults to ``os.name``.
Returns
-------
None
Example
-------
The following code runs the script ``root/module/make.py``.
.. code-block:: python
run_module(root = 'root', module = 'module')
"""
osname = osname if osname else os.name # https://github.com/sphinx-doc/sphinx/issues/759
try:
module_dir = os.path.join(root, module)
os.chdir(module_dir)
build_script = norm_path(build_script)
if not os.path.isfile(build_script):
raise CritError(messages.crit_error_no_file % build_script)
message = 'Running module `%s`' % module
message = format_message(message)
message = colored(message, attrs=['bold'])
print('\n' + message)
status = os.system(
'%s %s' %
(metadata.default_executables[osname]['python'], build_script))
if status != 0:
raise ProgramError()
except ProgramError:
sys.exit()
except:
error_message = 'Error with `run_module`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def add_log(*args, **kwargs):
"""Add log files in "*arg" list to "makelog" log file.
- If there are log files created inside scripts (for example
from a Perl or Python script), and we want to append their content
to the end of the default log file (usually '../output/make.log'), we
would use: `add_log('../output/analysis.log', '../output/another_log.log')`
- The number of log arguments can vary. If the log files don't actually exist,
errors will be printed to the common log file.
- If we want to append content of log files to a different log file than the
default, we would use:
`add_log('../output/analysis.log', '../output/another_log.log', makelog = '../different.log')`
"""
if 'makelog' in kwargs.keys():
makelog = kwargs['makelog']
else:
makelog = metadata.settings['makelog_file']
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise CritError(messages.crit_error_nomakelog % makelog)
print("\nAdd log file(s) to: ", makelog)
makelog = re.sub('\\\\', '/', makelog)
try:
LOGFILE = open(makelog, 'a')
except Exception as errmsg:
raise CritError((messages.crit_error_log % makelog) + '\n' +
str(errmsg))
try:
for log in args:
log = re.sub('\\\\', '/', log)
if not os.path.isfile(log):
print(messages.note_nofile % log, file=LOGFILE)
else:
LOGFILE.write(open(log, 'rU').read())
except:
print_error(LOGFILE)
LOGFILE.close()
def end_logging(LOGFILE, makelog, logtype):
time_end = datetime.datetime.now().replace(microsecond=0)
print(messages.note_logend % logtype, time_end, file=LOGFILE)
LOGFILE.close()
if not makelog: return
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise CritError(messages.crit_error_nomakelog % makelog)
MAKE_LOGFILE = open(makelog, 'a')
MAKE_LOGFILE.write(open(LOGFILE.name, 'rU').read())
MAKE_LOGFILE.close()
os.remove(LOGFILE.name)
def _parse_content(file, null):
""".. Parse content from input."""
with io.open(file, 'r', encoding='utf-8') as f:
content = f.readlines()
try:
tag = _parse_tag(content[0])
except:
raise_from(CritError(messages.crit_error_no_tag % file), None)
data = _parse_data(content[1:], null)
return (tag, data)
def _check_os(osname=os.name):
"""Check OS is either POSIX or NT.
Parameters
----------
osname : str, optional
Name of OS. Defaults to ``os.name``.
Returns
-------
None
"""
if osname not in ['posix', 'nt']:
raise CritError(messages.crit_error_unknown_system % osname)
def end_makelog(paths):
""".. End make log.
Appends to file ``makelog``, recording end time.
Note
----
We allow for writing to a make log even after the make log has ended. We do not recommend this for best practice.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
try:
makelog = get_path(paths, 'makelog')
if makelog:
makelog = norm_path(makelog)
message = 'Ending makelog file at: `%s`' % makelog
print(colored(message, metadata.color_success))
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise_from(CritError(messages.crit_error_no_makelog % makelog),
None)
with open(makelog, 'a', encoding='utf8') as MAKELOG:
time_end = str(datetime.datetime.now().replace(microsecond=0))
working_dir = os.getcwd()
print(messages.note_dash_line, file=MAKELOG)
print(messages.note_makelog_end + time_end, file=MAKELOG)
print(messages.note_working_directory + working_dir,
file=MAKELOG)
print(messages.note_dash_line, file=MAKELOG)
except:
error_message = 'Error with `end_makelog`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def write_log(self):
"""Write logs for shell command.
Returns
-------
None
"""
if self.makelog:
if not (metadata.makelog_started and os.path.isfile(self.makelog)):
raise CritError(messages.crit_error_no_makelog % self.makelog)
with open(self.makelog, 'a') as f:
print(self.output, file = f)
if self.log:
with open(self.log, 'w') as f:
f.write(self.output)
def input_to_array(filename):
# Import file
try:
FILENAME = open(filename, 'rU')
except:
raise CritError(messages.crit_error_file % filename)
# Delete header
filearray = []
for line in FILENAME:
if (not re.match('rev', line) and not re.match('linkpath', line)
and not re.match('\s*\#', line) and not re.match('\s*$', line)
and not re.match('url', line)):
filearray.append(line.rstrip('\n'))
FILENAME.close()
return filearray
def parse_file_list(self):
"""Parse wildcards in list of files.
Returns
-------
None
"""
self.file_list = convert_to_list(file_list, 'file')
file_list_parsed = [
f for file in self.file_list for f in glob.glob(file)
]
if file_list_parsed:
self.file_list = file_list_parsed
else:
error_list = [str(f) for f in self.file_list]
raise CritError(messages.crit_error_no_files % error_list)
def execute_command(self, command):
"""Execute shell command.
Parameters
----------
command : str
Shell command to execute.
Returns
-------
exit : tuple
Tuple (exit code, error message) for shell command.
"""
self.output = 'Executing command: `%s`' % command
print(colored(self.output, metadata.color_in_process))
try:
if not self.shell:
command = command.split()
process = subprocess_fix.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=self.shell,
universal_newlines=True)
process.wait()
stdout, stderr = process.communicate()
exit = (process.returncode, stderr)
if stdout:
self.output += '\n' + decode(stdout)
if stderr:
self.output += '\n' + decode(stderr)
pass
return (exit)
except:
error_message = messages.crit_error_bad_command % command
error_message = error_message + format_traceback()
raise_from(CritError(error_message), None)