def _create_copies(paths, file_list):
""".. Create copies from list of files containing copying instructions.
Create copies using instructions contained in files of list ``file_list``. Instructions are `string formatted <https://docs.python.org/3.4/library/string.html#format-string-syntax>`__ using paths dictionary ``paths``. Copies are written in directory ``move_dir``. Status messages are appended to file ``make log``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below. Dictionary additionally used to string format copying instructions.
file_list : str, list
File or list of files containing copying instructions.
Path Keys
---------
move_dir : str
Directory to write copies.
makelog : str
Path of makelog.
Returns
-------
source_map : list
List of (source, destination) for each copy created.
"""
move_dir = get_path(paths, 'move_dir')
move_list = MoveList(file_list, move_dir, paths)
if move_list.move_directive_list:
os.makedirs(move_dir)
source_map = move_list._create_copies()
else:
source_map = []
return (source_map)
def write_to_makelog(paths, message):
""".. Write to make log.
Appends string ``message`` to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
message : str
Message to append.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
makelog = get_path(paths, 'makelog')
if makelog:
makelog = norm_path(makelog)
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise_from(CritError(messages.crit_error_no_makelog % makelog), None)
with io.open(makelog, 'a', encoding = 'utf8', errors = 'ignore') as MAKELOG:
print(message, file = MAKELOG)
def update_internal_paths(paths):
""".. Update within-directory paths using default configuration file.
Returns dictionary ``paths`` with directory locations listed in file ``config``.
Parameters
----------
paths : dict
Dictionary of paths to update.
Dictionary should ex-ante contain values for all keys listed below.
Path Keys
---------
root : str
Path of project repo root
config : str
Path of user configuration file.
Returns
-------
paths : dict
Dictionary of paths.
"""
try:
config_default = get_path(paths, 'config')
config_default = open_yaml(config_default)
root = get_path(paths, 'root')
relative_paths = {path_label: os.path.join(root, path) for \
path_label, path in config_default['make_paths']['root_relative'].items()}
absolute_paths = config_default['make_paths']['absolute']
paths.update(relative_paths)
paths.update(absolute_paths)
return (paths)
except:
error_message = 'Error with update_external_paths. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def end_makelog(paths):
""".. End make log.
Appends to file ``makelog``, recording end time.
Note
----
We allow for writing to a make log even after the make log has ended. We do not recommend this for best practice.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
try:
makelog = get_path(paths, 'makelog')
if makelog:
makelog = norm_path(makelog)
message = 'Ending makelog file at: `%s`' % makelog
print(colored(message, metadata.color_success))
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise_from(CritError(messages.crit_error_no_makelog % makelog),
None)
with open(makelog, 'a', encoding='utf8') as MAKELOG:
time_end = str(datetime.datetime.now().replace(microsecond=0))
working_dir = os.getcwd()
print(messages.note_dash_line, file=MAKELOG)
print(messages.note_makelog_end + time_end, file=MAKELOG)
print(messages.note_working_directory + working_dir,
file=MAKELOG)
print(messages.note_dash_line, file=MAKELOG)
except:
error_message = 'Error with `end_makelog`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def start_makelog(paths):
""".. Start make log.
Writes file ``makelog``, recording start time. Sets make log status to boolean ``True``, which is used by other functions to confirm make log exists.
Note
----
The make log start condition is used by other functions to confirm a make log exists.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
try:
makelog = get_path(paths, 'makelog')
metadata.makelog_started = True
if makelog:
makelog = norm_path(makelog)
message = 'Starting makelog file at: `%s`' % makelog
print(colored(message, metadata.color_success))
with open(makelog, 'w', encoding='utf8') as MAKELOG:
time_start = str(
datetime.datetime.now().replace(microsecond=0))
working_dir = os.getcwd()
print(messages.note_dash_line, file=MAKELOG)
print(messages.note_makelog_start + time_start, file=MAKELOG)
print(messages.note_working_directory + working_dir,
file=MAKELOG)
print(messages.note_dash_line, file=MAKELOG)
except:
error_message = 'Error with `start_makelog`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def update_paths(paths):
""".. Update paths using user configuration file.
Updates dictionary ``paths`` with externals listed in file ``config_user``.
Note
----
The ``paths`` argument for :ref:`sourcing functions<sourcing functions>` is used not only to get
default paths for writing/logging, but also to
`string format <https://docs.python.org/3.4/library/string.html#format-string-syntax>`__
sourcing instructions.
Parameters
----------
paths : dict
Dictionary of paths to update.
Dictionary should ex-ante contain values for all keys listed below.
Path Keys
---------
config_user : str
Path of user configuration file.
Returns
-------
paths : dict
Dictionary of updated paths.
"""
try:
config_user = get_path(paths, 'config_user')
config_user = open_yaml(config_user)
if config_user['external']:
paths.update(config_user['external'])
return (paths)
except:
error_message = 'Error with update_paths. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def update_executables(paths, osname=None):
""".. Update executable names using user configuration file.
Updates executable names with executables listed in file ``config_user``.
Note
----
Executable names are used by :ref:`program functions <program functions>`.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
osname : str, optional
Name of OS. Defaults to ``os.name``.
Path Keys
---------
config_user : str
Path of user configuration file.
Returns
-------
None
"""
osname = osname if osname else os.name # https://github.com/sphinx-doc/sphinx/issues/759
try:
config_user = get_path(paths, 'config_user')
config_user = open_yaml(config_user)
_check_os(osname)
if config_user['local']['executables']:
metadata.default_executables[osname].update(
config_user['local']['executables'])
except:
error_message = 'Error with update_executables. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def run_stata(paths, program, **kwargs):
""".. Run Stata script using system command.
Runs script ``program`` using system command, with script specified
in the form of ``script.do``. Status messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
program : str
Path of script to run.
Path Keys
---------
makelog : str
Path of makelog.
Note
----
We recommend leaving all other parameters to their defaults.
Note
----
When a do-file contains a space in its name, different version of Stata save the
corresponding log file with different names. Some versions of Stata truncate the
name to everything before the first space of the do-file name.
Other Parameters
----------------
osname : str, optional
Name of OS. Used to determine syntax of system command. Defaults to ``os.name``.
shell : `bool`, optional
See `here <https://docs.python.org/3/library/subprocess.html#frequently-used-arguments>`_.
Defaults to ``True``.
log : str, optional
Path of program log. Program log is only written if specified.
Defaults to ``''`` (i.e., not written).
executable : str, optional
Executable to use for system command.
Defaults to executable specified in :ref:`default settings<default settings>`.
option : str, optional
Options for system command. Defaults to options specified in :ref:`default settings<default settings>`.
args : str, optional
Not applicable.
Returns
-------
None
Example
-------
.. code-block:: python
run_stata(paths, program = 'script.do')
"""
try:
makelog = get_path(paths, 'makelog')
direct = ProgramDirective(application='stata',
program=program,
makelog=makelog,
**kwargs)
# Get program output (partial)
program_name = direct.program.split(" ")[0]
program_name = os.path.split(program_name)[-1]
program_name = os.path.splitext(program_name)[0]
program_log_partial = os.path.join(os.getcwd(), program_name + '.log')
# Get program output (full)
program_log_full = os.path.join(os.getcwd(),
direct.program_name + '.log')
# Sanitize program
if direct.osname == "posix":
direct.program = re.escape(direct.program)
# Execute
command = metadata.commands[direct.osname]['stata'] % (
direct.executable, direct.option, direct.program)
exit_code, stderr = direct.execute_command(command)
if exit_code != 0:
error_message = 'Stata program executed with errors. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ProgramError(error_message, stderr), None)
try:
output = direct.move_program_output(program_log_partial,
direct.log)
except:
output = direct.move_program_output(program_log_full, direct.log)
_check_stata_output(output)
except ProgramError:
raise
except:
error_message = 'Error with `run_stata`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def write_source_logs(paths, source_map, depth=float('inf')):
""".. Write source logs.
Logs the following information for sources contained in list ``source_map``
(returned by :ref:`sourcing functions<sourcing functions>`).
- Mapping of symlinks/copies to sources (in file ``source_maplog``)
- Details on files contained in sources:
- File name (in file ``source_statslog``)
- Last modified (in file ``source_statslog``)
- File size (in file ``source_statslog``)
- File head (in file ``source_headlog``, optional)
When walking through sources, float ``depth`` determines level of depth to walk.
Status messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
source_map : list
Mapping of symlinks/copies (destination) to sources (returned by :ref:`sourcing functions<sourcing functions>`).
depth : float, optional
Level of depth when walking through source directories. Defaults to infinite.
Path Keys
---------
source_statslog : str
Path to write source statistics log.
source_headslog : str, optional
Path to write source headers log.
source_maplog : str
Path to write source map log.
makelog : str
Path of makelog.
Returns
-------
None
Example
-------
The following code will log information for all files listed in ``source_map``.
Therefore, files contained in directories listed in ``source_map`` will be ignored.
.. code-block:: python
write_source_logs(paths, depth = 1)
The following code will log information for all files listed in ``source_map``
and any file in all directories listed in ``source_map``, regardless of level of subdirectory.
.. code-block :: python
write_source_logs(paths, depth = float('inf'))
"""
try:
source_statslog = get_path(paths, 'source_statslog')
source_headslog = get_path(paths, 'source_headslog', throw_error=False)
source_maplog = get_path(paths, 'source_maplog')
source_list = [source for source, destination in source_map]
source_list = [glob_recursive(source, depth) for source in source_list]
source_files = [f for source in source_list for f in source]
source_files = set(source_files)
# ACTION: DECIDE WHETHER TO ALLOW FOR RAW DIRECTORY
raw_dir = get_path(paths, 'raw_dir', throw_error=False)
if raw_dir:
raw_files = glob_recursive(raw_dir)
source_files = set(source_files + raw_files)
if source_statslog:
source_statslog = norm_path(source_statslog)
_write_stats_log(source_statslog, source_files)
if source_headslog:
source_headslog = norm_path(source_headslog)
_write_heads_log(source_headslog, source_files)
if source_maplog:
source_maplog = norm_path(source_maplog)
_write_source_maplog(source_maplog, source_map)
message = 'Source logs successfully written!'
write_to_makelog(paths, message)
print(colored(message, metadata.color_success))
except:
error_message = 'Error with `write_source_logs`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def log_files_in_output(paths,
depth = float('inf')):
""".. Log files in output directory.
Logs the following information for all files contained in directory ``output_dir``.
- File name (in file ``output_statslog``)
- Last modified (in file ``output_statslog``)
- File size (in file ``output_statslog``)
- File head (in file ``output_headslog``, optional)
When walking through directory ``output_dir``, float ``depth`` determines level of depth to walk.
Status messages are appended to file ``makelog``.
Include additional output directories to walk through
(typically directories that you wish to keep local) in directory list ``output_local_dir``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
depth : float, optional
Level of depth when walking through output directory. Defaults to infinite.
Path Keys
---------
output_dir : str
Path of output directory.
output_local_dir : str, list, optional
Path or list of paths of local output directories. Defaults to ``[]`` (i.e., none).
output_statslog : str
Path to write output statistics log.
output_headslog : str, optional
Path to write output headers log.
makelog : str
Path of makelog.
Returns
-------
None
Example
-------
The following code will log information for all files contained in
only the first level of ``paths['output_dir']``.
Therefore, files contained in subdirectories will be ignored.
.. code-block:: python
log_files_in_outputs(paths, depth = 1)
The following code will log information for any file in ``paths['output_dir']``,
regardless of level of subdirectory.
.. code-block :: python
log_files_in_outputs(paths, depth = float('inf'))
"""
try:
output_dir = get_path(paths, 'output_dir')
output_local_dir = get_path(paths, 'output_local_dir', throw_error = False)
output_statslog = get_path(paths, 'output_statslog')
output_headslog = get_path(paths, 'output_headslog', throw_error = False)
if output_local_dir:
output_local_dir = convert_to_list(output_local_dir, 'dir')
else:
output_local_dir = []
output_files = glob_recursive(output_dir, depth)
output_local_files = [f for dir_path in output_local_dir for f in glob_recursive(dir_path, depth)]
output_files = set(output_files + output_local_files)
if output_statslog:
output_statslog = norm_path(output_statslog)
_write_stats_log(output_statslog, output_files)
if output_headslog:
output_headslog = norm_path(output_headslog)
_write_heads_log(output_headslog, output_files)
message = 'Output logs successfully written!'
write_to_makelog(paths, message)
print(colored(message, metadata.color_success))
except:
error_message = 'Error with `log_files_in_output`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths, error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def run_jupyter(paths, program, timeout=None, kernel_name=''):
"""Run Jupyter notebook using system command.
Runs notebook ``program`` using Python API, with notebook specified in the form of ``notebook.ipynb``. Status messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
program : str
Path of script to run.
Path Keys
---------
makelog : str
Path of makelog.
Note
----
We recommend leaving all other parameters to their defaults.
Other Parameters
----------------
timeout : int, optional
Time to wait (in seconds) to finish executing a cell before raising exception. Defaults to no timeout.
kernel_name : str, optional
Name of kernel to use for execution (e.g., ``python2`` for standard Python 2 kernel, ``python3`` for standard Python 3 kernel). Defaults to ``''`` (i.e., kernel specified in notebook).
Returns
-------
None
Example
-------
.. code-block:: python
run_jupyter(paths, program = 'notebook.ipynb')
"""
try:
makelog = get_path(paths, 'makelog')
program = norm_path(program)
with open(program) as f:
message = 'Processing notebook: `%s`' % program
write_to_makelog(paths, message)
print(colored(message, 'cyan'))
if not kernel_name:
kernel_name = 'python%s' % sys.version_info[0]
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name)
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {'metadata': {'path': '.'}})
with open(program, 'wt') as f:
nbformat.write(nb, f)
except:
error_message = 'Error with `run_jupyter`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def link_inputs(paths, file_list):
""".. Create symlinks to inputs from list of files containing linking instructions.
Create symbolic links using instructions contained in files of list ``file_list``. Instructions are `string formatted <https://docs.python.org/3.4/library/string.html#format-string-syntax>`__ using paths dictionary ``paths``. Symbolic links are written in directory ``input_dir``. Status messages are appended to file ``make log``.
Instruction files on how to create symbolic links (destinations) from targets (sources) should be formatted in the following way.
.. code-block:: md
# Each line of instruction should contain a destination and source delimited by a `|`
# Lines beginning with # are ignored
destination | source
.. Note::
Symbolic links can be created to both files and directories.
.. Note::
Instruction files can be specified with the * shell pattern (see `here <https://www.gnu.org/software/findutils/manual/html_node/find_html/Shell-Pattern-Matching.html>`__). Destinations and their sources can also be specified with the * shell pattern. The number of wildcards must be the same for both destinations and sources.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below. Dictionary additionally used to string format linking instructions.
file_list : str, list
File or list of files containing linking instructions.
Path Keys
---------
input_dir : str
Directory to write symlinks.
makelog : str
Path of makelog.
Returns
-------
source_map : list
List of (source, destination) for each symlink created.
Example
-------
Suppose you call the following function.
.. code-block:: python
link_inputs(paths, ['file1'], formatting_dict)
Suppose ``paths`` contained the following values.
.. code-block:: md
paths = {'root': '/User/root/',
'makelog': 'make.log',
'input_dir': 'input'}
Now suppose instruction file ``file1`` contained the following text.
.. code-block:: md
destination1 | {root}/source1
The ``{root}`` in the instruction file would be string formatted using ``paths``. Therefore, the function would parse the instruction as:
.. code-block:: md
destination1 | /User/root/source1
Example
-------
The following code would use instruction files ``file1`` and ``file2`` to create symbolic links.
.. code-block:: python
link_inputs(paths, ['file1', 'file2'])
Suppose instruction file ``file1`` contained the following text.
.. code-block:: md
destination1 | source1
destination2 | source2
Symbolic links ``destination1`` and ``destination1`` would be created in directory ``paths['input_dir']``. Their targets would be ``source1`` and ``source2``, respectively.
Example
-------
Suppose you have the following targets.
.. code-block:: md
source1
source2
source3
Specifying ``destination* | source*`` in one of your instruction files would create the following symbolic links in ``paths['input_dir']``.
.. code-block:: md
destination1
destination2
destination3
"""
try:
paths['move_dir'] = get_path(paths, 'input_dir')
source_map = _create_links(paths, file_list)
message = 'Input links successfully created!'
write_to_makelog(paths, message)
print(colored(message, metadata.color_success))
return (source_map)
except:
error_message = 'An error was encountered with `link_inputs`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def execute_command(paths, command, **kwargs):
""".. Run system command.
Runs system command `command` with shell execution boolean ``shell``.
Outputs are appended to file ``makelog`` and written to system command log file ``log``.
Status messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
command : str
System command to run.
shell : `bool`, optional
See `here <https://docs.python.org/3/library/subprocess.html#frequently-used-arguments>`_.
Defaults to ``True``.
log : str, optional
Path of system command log. System command log is only written if specified.
Defaults to ``''`` (i.e., not written).
Path Keys
---------
makelog : str
Path of makelog.
Note
----
We recommend leaving all other parameters to their defaults.
Other Parameters
----------------
osname : str, optional
Name of OS. Used to check if OS is supported. Defaults to ``os.name``.
Returns
-------
None
Example
-------
The following code executes the ``ls`` command,
writes outputs to system command log file ``'file'``,
and appends outputs and/or status messages to ``paths['makelog']``.
.. code-block:: python
execute_command(paths, 'ls', log = 'file')
"""
try:
makelog = get_path(paths, 'makelog')
direct = Directive(makelog=makelog, **kwargs)
# Execute
exit_code, stderr = direct.execute_command(command)
direct.write_log()
if exit_code != 0:
error_message = 'Command executed with errors. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ProgramError(error_message, stderr), None)
except ProgramError:
raise
except:
error_message = 'Error with `execute_command`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def run_lyx(paths, program, doctype='', **kwargs):
""".. Run LyX script using system command.
Compiles document ``program`` using system command, with document specified
in the form of ``script.lyx``. Status messages are appended to file ``makelog``.
PDF outputs are written in directory ``output_dir``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
program : str
Path of script to run.
doctype : str, optional
Type of LyX document. Takes either ``'handout'`` and ``'comments'``.
All other strings will default to standard document type.
Defaults to ``''`` (i.e., standard document type).
Path Keys
---------
makelog : str
Path of makelog.
output_dir : str
Directory to write PDFs.
Note
----
We recommend leaving all other parameters to their defaults.
Other Parameters
----------------
osname : str, optional
Name of OS. Used to determine syntax of system command. Defaults to ``os.name``.
shell : `bool`, optional
See `here <https://docs.python.org/3/library/subprocess.html#frequently-used-arguments>`_.
Defaults to ``True``.
log : str, optional
Path of program log. Program log is only written if specified.
Defaults to ``''`` (i.e., not written).
executable : str, optional
Executable to use for system command.
Defaults to executable specified in :ref:`default settings<default settings>`.
option : str, optional
Options for system command. Defaults to options specified in :ref:`default settings<default settings>`.
args : str, optional
Not applicable.
Returns
-------
None
Example
-------
.. code-block:: python
run_lyx(paths, program = 'script.lyx')
"""
try:
makelog = get_path(paths, 'makelog')
output_dir = get_path(paths, 'output_dir')
direct = LyXDirective(output_dir=output_dir,
doctype=doctype,
application='lyx',
program=program,
makelog=makelog,
**kwargs)
# Make handout/comments LyX file
if direct.doctype:
temp_name = os.path.join(direct.program_name + '_' +
direct.doctype)
temp_program = os.path.join(direct.program_dir, temp_name + '.lyx')
beamer = False
shutil.copy2(direct.program, temp_program)
for line in fileinput.input(temp_program,
inplace=True,
backup='.bak'):
if r'\textclass beamer' in line:
beamer = True
if direct.doctype == 'handout' and beamer and (r'\options'
in line):
line = line.rstrip('\n') + ', handout\n'
elif direct.doctype == 'comments' and (
r'\begin_inset Note Note' in line):
line = line.replace('Note Note', 'Note Greyedout')
print(line)
else:
temp_name = direct.program_name
temp_program = direct.program
# Execute
command = metadata.commands[direct.osname][direct.application] % (
direct.executable, direct.option, temp_program)
exit_code, stderr = direct.execute_command(command)
direct.write_log()
if exit_code != 0:
error_message = 'LyX program executed with errors. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ProgramError(error_message, stderr), None)
# Move PDF output
temp_pdf = os.path.join(direct.program_dir, temp_name + '.pdf')
output_pdf = os.path.join(direct.output_dir,
direct.program_name + '.pdf')
if temp_pdf != output_pdf:
shutil.copy2(temp_pdf, output_pdf)
os.remove(temp_pdf)
# Remove handout/comments LyX file
if direct.doctype:
os.remove(temp_program)
except ProgramError:
raise
except:
error_message = 'Error with `run_lyx`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def check_module_size(paths):
""".. Check file sizes for module.
Checks file sizes for files to be committed in the current working directory.
Compares file sizes to size limits in file ``config`` and
produces warnings if any of the following limits are exceeded.
- Individual size of a file tracked by git lfs (``file_MB_limit_lfs``)
- Total size of all files tracked by git lfs (``total_MB_limit_lfs``)
- Individual size of a file tracked by git (``file_MB_limit``)
- Total size of all files tracked by git (``total_MB_limit``)
Warning messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
Path Keys
---------
config : str
Path of project configuration file.
makelog : str
Path of makelog.
Returns
-------
None
"""
try:
git_files, git_lfs_files = _get_dir_sizes('.')
file_MB, total_MB, file_MB_lfs, total_MB_lfs = _get_size_values(
git_files, git_lfs_files)
config = get_path(paths, 'config')
config = open_yaml(config)
max_file_sizes = config['max_file_sizes']
print_message = ''
if file_MB > max_file_sizes['file_MB_limit']:
print_message = print_message + messages.warning_git_file_print % max_file_sizes[
'file_MB_limit']
if total_MB > max_file_sizes['total_MB_limit']:
print_message = print_message + messages.warning_git_repo % max_file_sizes[
'total_MB_limit']
if file_MB_lfs > max_file_sizes['file_MB_limit_lfs']:
print_message = print_message + messages.warning_git_lfs_file_print % max_file_sizes[
'file_MB_limit_lfs']
if total_MB_lfs > max_file_sizes['total_MB_limit_lfs']:
print_message = print_message + messages.warning_git_lfs_repo % max_file_sizes[
'total_MB_limit_lfs']
print_message = print_message.strip()
log_message = ''
if file_MB > max_file_sizes['file_MB_limit']:
log_message = log_message + messages.warning_git_file_log % max_file_sizes[
'file_MB_limit']
exceed_files = [
f for (f, s) in git_files.items()
if s / (1024**2) > max_file_sizes['file_MB_limit']
]
exceed_files = '\n'.join(exceed_files)
log_message = log_message + '\n' + exceed_files
if total_MB > max_file_sizes['total_MB_limit']:
log_message = log_message + messages.warning_git_repo % max_file_sizes[
'total_MB_limit']
if file_MB_lfs > max_file_sizes['file_MB_limit_lfs']:
log_message = log_message + messages.warning_git_lfs_file_log % max_file_sizes[
'file_MB_limit_lfs']
exceed_files = [
f for (f, s) in git_lfs_files.items()
if s / (1024**2) > max_file_sizes['file_MB_limit_lfs']
]
exceed_files = '\n'.join(exceed_files)
log_message = log_message + '\n' + exceed_files
if total_MB_lfs > max_file_sizes['total_MB_limit_lfs']:
log_message = log_message + messages.warning_git_lfs_repo % max_file_sizes[
'total_MB_limit_lfs']
log_message = log_message.strip()
if print_message:
print(colored(print_message, metadata.color_failure))
if log_message:
write_to_makelog(paths, log_message)
except:
error_message = 'Error with `check_repo_size`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def run_stat_transfer(paths, program, **kwargs):
""".. Run StatTransfer script using system command.
Runs script ``program`` using system command, with script specified
in the form of ``script.stc`` or ``script.stcmd``.
Status messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
program : str
Path of script to run.
Path Keys
---------
makelog : str
Path of makelog.
Note
----
We recommend leaving all other parameters to their defaults.
Other Parameters
----------------
osname : str, optional
Name of OS. Used to determine syntax of system command. Defaults to ``os.name``.
shell : `bool`, optional
See `here <https://docs.python.org/3/library/subprocess.html#frequently-used-arguments>`_.
Defaults to ``True``.
log : str, optional
Path of program log. Program log is only written if specified.
Defaults to ``''`` (i.e., not written).
executable : str, optional
Executable to use for system command.
Defaults to executable specified in :ref:`default settings<default settings>`.
option : str, optional
Options for system command. Defaults to options specified in :ref:`default settings<default settings>`.
args : str, optional
Not applicable.
Returns
-------
None
Example
-------
.. code-block:: python
run_stat_transfer(paths, program = 'script.stc')
"""
try:
makelog = get_path(paths, 'makelog')
direct = ProgramDirective(application='st',
program=program,
makelog=makelog,
**kwargs)
# Execute
command = metadata.commands[direct.osname][direct.application] % (
direct.executable, direct.program)
exit_code, stderr = direct.execute_command(command)
direct.write_log()
if exit_code != 0:
error_message = 'StatTransfer program executed with errors. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ProgramError(error_message, stderr), None)
except ProgramError:
raise
except:
error_message = 'Error with `run_stat_transfer`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def run_latex(paths, program, **kwargs):
""".. Run LaTeX script using system command.
Compiles document ``program`` using system command, with document specified
in the form of ``script.tex``. Status messages are appended to file ``makelog``.
PDF outputs are written in directory ``output_dir``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
program : str
Path of script to run.
Path Keys
---------
makelog : str
Path of makelog.
output_dir : str
Directory to write PDFs.
Note
----
We recommend leaving all other parameters to their defaults.
Note
----
This function creates and removes a directory named ``latex_auxiliary_dir``.
Other Parameters
----------------
osname : str, optional
Name of OS. Used to determine syntax of system command. Defaults to ``os.name``.
shell : `bool`, optional
See `here <https://docs.python.org/3/library/subprocess.html#frequently-used-arguments>`_.
Defaults to ``True``.
log : str, optional
Path of program log. Program log is only written if specified.
Defaults to ``''`` (i.e., not written).
executable : str, optional
Executable to use for system command.
Defaults to executable specified in :ref:`default settings<default settings>`.
option : str, optional
Options for system command. Defaults to options specified in :ref:`default settings<default settings>`.
args : str, optional
Not applicable.
Returns
-------
None
Example
-------
.. code-block:: python
run_latex(paths, program = 'script.tex')
"""
try:
makelog = get_path(paths, 'makelog')
output_dir = get_path(paths, 'output_dir')
direct = LyXDirective(output_dir=output_dir,
application='latex',
program=program,
makelog=makelog,
**kwargs)
temp_name = direct.program_name
temp_program = direct.program
# Generate folder for auxiliary files
os.mkdir('latex_auxiliary_dir')
# Execute
command = metadata.commands[direct.osname][direct.application] % (
direct.executable, direct.option, temp_program)
exit_code, stderr = direct.execute_command(command)
direct.write_log()
if exit_code != 0:
error_message = 'LaTeX program executed with errors. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ProgramError(error_message, stderr), None)
# Move PDF output
temp_pdf = os.path.join('latex_auxiliary_dir', temp_name + '.pdf')
output_pdf = os.path.join(direct.output_dir,
direct.program_name + '.pdf')
if temp_pdf != output_pdf:
shutil.copy2(temp_pdf, output_pdf)
shutil.rmtree('latex_auxiliary_dir')
# Remove auxiliary files
except ProgramError:
raise
except:
error_message = 'Error with `run_latex`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)