def get_paths(self):
"""Normalize paths.
Returns
-------
None
"""
self.makelog = norm_path(self.makelog)
self.log = norm_path(self.log)
def zip_dir(source_dir, zip_dest):
""".. Zip directory to file.
Zips directory ``source_dir`` to file ``zip_dest``.
Parameters
----------
source_dir : str
Path of directory to zip.
zip_dest : str
Destination of zip file.
Returns
-------
None
"""
try:
with zipfile.ZipFile('%s' % (zip_dest), 'w', zipfile.ZIP_DEFLATED, allowZip64 = True) as z:
source_dir = norm_path(source_dir)
for root, dirs, files in os.walk(source_dir):
for f in files:
file_path = os.path.join(root, f)
file_name = os.path.basename(file_path)
z.write(file_path, file_name)
message = 'Zipped: `%s` as `%s`' % (file_path, file_name)
print(colored(message, metadata.color_success))
except:
error_message = 'Error with `zip_dir`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def _get_file_sizes(dir_path, exclude):
""".. Walk through directory and get file sizes.
Get file sizes for files in directory ``dir_path``, ignoring subdirectories in list ``exclude``.
Parameters
----------
dir_path : str
Path of directory to walk through.
exclude : list
List of subdirectories to exclude when walking.
Returns
-------
file_size : dict
Dictionary of ``{file : size}`` for each file in ``dir_path``.
"""
file_sizes = []
for root, dirs, files in os.walk(dir_path, topdown=True):
dirs[:] = [d for d in dirs if d not in exclude]
files = [os.path.join(root, f) for f in files]
files = [norm_path(f) for f in files]
sizes = [os.lstat(f).st_size for f in files]
file_sizes.extend(zip(files, sizes))
file_sizes = dict(file_sizes)
return (file_sizes)
def write_to_makelog(paths, message):
""".. Write to make log.
Appends string ``message`` to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
message : str
Message to append.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
makelog = get_path(paths, 'makelog')
if makelog:
makelog = norm_path(makelog)
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise_from(CritError(messages.crit_error_no_makelog % makelog), None)
with io.open(makelog, 'a', encoding = 'utf8', errors = 'ignore') as MAKELOG:
print(message, file = MAKELOG)
def _get_git_status(repo):
""".. Get git status.
Get git status for repository ``repo``.
Parameters
----------
repo : :class:`git.Repo `
Git repository to show working tree status.
Returns
-------
file_list : list
List of changed files in git repository according to git status.
"""
root = repo.working_tree_dir
file_list = repo.git.status(porcelain=True)
file_list = file_list.split('\n')
file_list = [f.lstrip().lstrip('MADRCU?!').lstrip() for f in file_list]
file_list = [os.path.join(root, f) for f in file_list]
file_list = [norm_path(f) for f in file_list]
return (file_list)
def run_module(root, module, build_script='make.py', osname=None):
""".. Run module.
Runs script `build_script` in module directory `module` relative to root of repository `root`.
Parameters
----------
root : str
Directory of root.
module: str
Name of module.
build_script : str
Name of build script. Defaults to ``make.py``.
osname : str, optional
Name of OS. Used to determine syntax of system command. Defaults to ``os.name``.
Returns
-------
None
Example
-------
The following code runs the script ``root/module/make.py``.
.. code-block:: python
run_module(root = 'root', module = 'module')
"""
osname = osname if osname else os.name # https://github.com/sphinx-doc/sphinx/issues/759
try:
module_dir = os.path.join(root, module)
os.chdir(module_dir)
build_script = norm_path(build_script)
if not os.path.isfile(build_script):
raise CritError(messages.crit_error_no_file % build_script)
message = 'Running module `%s`' % module
message = format_message(message)
message = colored(message, attrs=['bold'])
print('\n' + message)
status = os.system(
'%s %s' %
(metadata.default_executables[osname]['python'], build_script))
if status != 0:
raise ProgramError()
except ProgramError:
sys.exit()
except:
error_message = 'Error with `run_module`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def parse_program(self):
"""Parse program for directory, name, and extension.
Returns
-------
None
"""
self.program = norm_path(self.program)
self.program_dir = os.path.dirname(self.program)
self.program_base = os.path.basename(self.program)
self.program_name, self.program_ext = os.path.splitext(self.program_base)
def remove_path(path, option = '', quiet = False):
""".. Remove path using system command.
Remove path ``path`` using system command. Safely removes symbolic links.
Path can be specified with the * shell pattern
(see `here <https://www.gnu.org/software/findutils/manual/html_node/find_html/Shell-Pattern-Matching.html>`__).
Parameters
----------
path : str
Path to remove.
option : str, optional
Options for system command. Defaults to ``-rf`` for POSIX and ``/s /q`` for NT.
quiet : bool, optional
Suppress printing of path removed. Defaults to ``False``.
Returns
-------
None
Example
-------
The following code removes path ``path``.
.. code-block:: python
remove_path('path')
The following code removes all paths beginning with ``path``.
.. code-block:: python
remove_path('path*')
"""
try:
path = norm_path(path)
if not option:
option = metadata.default_options[os.name]['rmdir']
command = metadata.commands[os.name]['rmdir'] % (option, path)
process = subprocess_fix.Popen(command, shell = True)
process.wait()
# ACTION ITEM: ADD DEBUGGING TO SUBPROCESS CALL
if not quiet:
message = 'Removed: `%s`' % path
print(colored(message, metadata.color_success))
except:
error_message = 'Error with `remove_path`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def get_paths(self):
"""Parse sources and destinations from line.
Returns
-------
None
"""
try:
self.line = self.line.split('|')
self.line = [l.strip() for l in self.line]
self.line = [l.strip('"\'') for l in self.line]
self.destination, self.source = self.line
except Exception:
error_message = messages.crit_error_bad_move % (self.raw_line,
self.file)
error_message = error_message + format_traceback()
raise_from(CritError(error_message), None)
self.source = norm_path(self.source)
self.destination = norm_path(
os.path.join(self.move_dir, self.destination))
def copy_output(file, copy_dir):
""".. Copy output file.
Copies output ``file`` to directory ``copy_dir`` with user prompt to confirm copy.
Parameters
----------
file : str
Path of file to copy.
copy_dir : str
Directory to copy file.
Returns
-------
None
"""
file = norm_path(file)
copy_dir = norm_path(copy_dir)
message = colored(messages.warning_copy, color='cyan')
upload = input(message % (file, copy_dir))
if upload.lower().strip() == "yes":
shutil.copy(file, copy_dir)
def end_makelog(paths):
""".. End make log.
Appends to file ``makelog``, recording end time.
Note
----
We allow for writing to a make log even after the make log has ended. We do not recommend this for best practice.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
try:
makelog = get_path(paths, 'makelog')
if makelog:
makelog = norm_path(makelog)
message = 'Ending makelog file at: `%s`' % makelog
print(colored(message, metadata.color_success))
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise_from(CritError(messages.crit_error_no_makelog % makelog),
None)
with open(makelog, 'a', encoding='utf8') as MAKELOG:
time_end = str(datetime.datetime.now().replace(microsecond=0))
working_dir = os.getcwd()
print(messages.note_dash_line, file=MAKELOG)
print(messages.note_makelog_end + time_end, file=MAKELOG)
print(messages.note_working_directory + working_dir,
file=MAKELOG)
print(messages.note_dash_line, file=MAKELOG)
except:
error_message = 'Error with `end_makelog`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def remove_dir(dir_list, quiet = False):
""".. Remove directory using system command.
Remove directories in list ``dir_list`` using system command.
Safely removes symbolic links. Directories can be specified with the * shell pattern
(see `here <https://www.gnu.org/software/findutils/manual/html_node/find_html/Shell-Pattern-Matching.html>`__).
Parameters
----------
dir_list : str, list
Directory or list of directories to remove.
quiet : bool, optional
Suppress printing of directories removed. Defaults to ``False``.
Returns
-------
None
Example
-------
The following code removes directories ``dir1`` and ``dir2``.
.. code-block:: python
remove_dir(['dir1', 'dir2'])
The following code removes directories beginning with ``dir``.
.. code-block:: python
remove_dir(['dir1*'])
"""
try:
dir_list = convert_to_list(dir_list, 'dir')
dir_list = [norm_path(dir_path) for dir_path in dir_list]
dir_list = [d for directory in dir_list for d in glob.glob(directory)]
for dir_path in dir_list:
if os.path.isdir(dir_path):
remove_path(dir_path, quiet = quiet)
elif os.path.isfile(dir_path):
raise_from(TypeError(messages.type_error_not_dir % dir_path), None)
except:
error_message = 'Error with `remove_dir`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def move_program_output(self, program_output, log_file=''):
"""Move program outputs.
Notes
-----
Certain applications create program outputs that need to be moved to
appropriate logging files.
Parameters
----------
program_output : str
Path of program output.
log_file : str, optional
Path of log file. Log file is only written if specified.
Defaults to ``''`` (i.e., not written).
"""
program_output = norm_path(program_output)
try:
with io.open(program_output,
'r',
encoding='utf-8',
errors='ignore') as f:
out = f.read()
except:
error_message = messages.crit_error_no_program_output % (
program_output, self.program)
error_message = error_message + format_traceback()
raise_from(CritError(error_message), None)
if self.makelog:
if not (metadata.makelog_started and os.path.isfile(self.makelog)):
raise CritError(messages.crit_error_no_makelog % self.makelog)
with io.open(self.makelog, 'a', encoding='utf-8',
errors='ignore') as f:
print(out, file=f)
if log_file:
if program_output != log_file:
shutil.copy2(program_output, log_file)
os.remove(program_output)
else:
os.remove(program_output)
return (out)
def start_makelog(paths):
""".. Start make log.
Writes file ``makelog``, recording start time. Sets make log status to boolean ``True``, which is used by other functions to confirm make log exists.
Note
----
The make log start condition is used by other functions to confirm a make log exists.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
Path Keys
---------
makelog : str
Path of makelog.
Returns
-------
None
"""
try:
makelog = get_path(paths, 'makelog')
metadata.makelog_started = True
if makelog:
makelog = norm_path(makelog)
message = 'Starting makelog file at: `%s`' % makelog
print(colored(message, metadata.color_success))
with open(makelog, 'w', encoding='utf8') as MAKELOG:
time_start = str(
datetime.datetime.now().replace(microsecond=0))
working_dir = os.getcwd()
print(messages.note_dash_line, file=MAKELOG)
print(messages.note_makelog_start + time_start, file=MAKELOG)
print(messages.note_working_directory + working_dir,
file=MAKELOG)
print(messages.note_dash_line, file=MAKELOG)
except:
error_message = 'Error with `start_makelog`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def _get_git_ignore(repo):
""".. Get files ignored by git.
Get files ignored by git for repository ``repo``.
Parameters
----------
repo : :class:`git.Repo`
Git repository to get ignored files.
Returns
-------
ignore_files : list
List of files in repository ignored by git.
"""
g = git.Git(repo)
root = repo.working_tree_dir
ignore = g.execute('git status --porcelain --ignored',
shell=True).split('\n')
ignore = [i for i in ignore if re.match('!!', i)]
ignore = [i.lstrip('!!').strip() for i in ignore]
ignore = [os.path.join(root, i) for i in ignore]
ignore_dirs = []
ignore_files = []
for i in ignore:
if os.path.isdir(i):
ignore_dirs.append(i)
elif os.path.isfile(i):
ignore_files.append(i)
for i in ignore_dirs:
for root, dirs, files in os.walk(i):
files = [os.path.join(root, f) for f in files]
ignore_files.extend(files)
ignore_files = [norm_path(i) for i in ignore_files]
return (ignore_files)
def parse_file_list(self):
"""Parse wildcards in list of files.
Returns
-------
None
"""
if self.file_list:
self.file_list = convert_to_list(self.file_list, 'file')
self.file_list = [norm_path(file) for file in self.file_list]
file_list_parsed = [
f for file in self.file_list for f in glob.glob(file)
]
if file_list_parsed:
self.file_list = file_list_parsed
else:
error_list = [decode(f) for f in self.file_list]
raise CritError(messages.crit_error_no_files % error_list)
def _insert_tables(template, tables, null):
""".. Fill tables for template.
Parameters
----------
template : str
Path of template to fill.
tables : dict
Dictionary ``{tag: values}`` of tables.
Returns
-------
template : str
Filled template.
"""
template = norm_path(template)
if re.search('\.lyx', template):
doc = _insert_tables_lyx(template, tables, null)
elif re.search('\.tex', template):
doc = _insert_tables_latex(template, tables, null)
return (doc)
def tablefill(inputs, template, output, null='.'):
""".. Fill tables for template using inputs.
Fills tables in document ``template`` using files in list ``inputs``.
Writes filled document to file ``output``.
Null characters in ``inputs`` are replaced with value ``null``.
Parameters
----------
inputs : list
Input or list of inputs to fill into template.
template : str
Path of template to fill.
output : str
Path of output.
null : str
Value to replace null characters (i.e., ``''``, ``'.'``, ``'NA'``). Defaults to ``'.'``.
Returns
-------
None
Example
-------
.. code-block::
#################################################################
# tablefill_readme.txt - Help/Documentation for tablefill.py
#################################################################
Description:
tablefill.py is a Python module designed to fill LyX/Tex tables with output
from text files (usually output from Stata or Matlab).
Usage:
Tablefill takes as input a LyX (or Tex) file containing empty tables (the template
file) and text files containing data to be copied to these tables (the
input files), and produces a LyX (or Tex) file with filled tables (the output file).
For brevity, LyX will be used to denote LyX or Tex files throughout.
Tablefill must first be imported to make.py. This is typically achieved
by including the following lines:
```
from gslab_fill.tablefill import tablefill
```
Once the module has been imported, the syntax used to call tablefill is
as follows:
```
tablefill(input = 'input_file(s)', template = 'template_file',
output = 'output_file')
```
The argument 'template' is the user written LyX file which contains the
tables to be filled in. The argument 'input' is a list of the text files
containing the output to be copied to the LyX tables. If there are multiple
input text files, they are listed as: input = 'input_file_1 input_file_2'.
The argument 'output' is the name of the filled LyX file to be produced.
Note that this file is created by tablefill.py and should not be edited
manually by the user.
###########################
Input File Format:
###########################
The data needs to be tab-delimited rows of numbers (or characters),
preceeded by `<label>`. The < and > are mandatory. The numbers can be
arbitrarily long, can be negative, and can also be in scientific notation.
Examples:
----------
```
<tab:Test>
1 2 3
2 3 1
3 1 2
```
```
<tab:FunnyMat>
1 2 3 23 2
2 3
3 1 2 2
1
```
(The rows do not need to be of equal length.)
Completely blank (no tab) lines are ignored.
If a "cell" is merely "." or "[space]", then it is treated as completely
missing. That is, in the program:
```
<tab:Test>
1 2 3
2 . 1 3
3 1 2
```
is equivalent to:
```
<tab:Test>
1 2 3
2 1 3
3 1 2
```
This feature is useful as Stata outputs missing values in numerical
variables as ".", and missing values in string variables as "[space]".
................................
Scientific Notation Notes:
................................
The scientific notation ihas to be of the form:
[numbers].[numbers]e(+/-)[numbers]
Examples:
```
23.2389e+23
-2.23e-2
-0.922e+3
```
###########################
Template LyX Format:
###########################
The LyX template file determines where the numbers from the input files are placed.
Every table in the template file (if it is to be filled) must appear within a float.
There must be one, and only one, table object inside the float, and the table name
must include a label object that corresponds to the label of the required table in
the input file.
Note that table names cannot be duplicated. For a single template file, each table
to be filled must have a unique label, and there must be one, and only one, table with
that same label in the text files used as input. Having multiple tables with the
same name in the input files or in the template file will cause errors.
Note also that labels are NOT case-sensitive. That is, <TAB:Table1> is considered
the same as `<tab:table1>`.
In the LyX tables, "cells" to be filled with entries from the input text files are
indicated by the following tags:
`"###" (no quotes)`
or
`"#[number][,]#" (no quotes)`
The first case will result in a literal substitution. I.e. whatever is in the text
tables for that cell will be copied over. The second case will convert the data
table's number (if in scientific notation) and will truncate this converted number
to [number] decimal places. It will automatically round while doing so.
If a comma appears after the number (within #[number]#), then it will add commas
to the digits to the left of the decimal place.
Examples:
---------
```
2309.2093 + ### = 2309.2093
2309.2093 + #4# = 2309.2093
2309.2093 + #5# = 2309.20930
2309.2093 + #20# = 2309.20930000000000000000
2309.2093 + #3# = 2309.209
2309.2093 + #2# = 2309.21
2309.2093 + #0# = 2309
2309.2093 + #0,# = 2,309
```
```
-2.23e-2 + #2# = -0.0223 + #2# = -0.02
-2.23e-2 + #7# = -0.0223 + #7# = -0.0223000
-2.23e+10 + #7,# = -22300000000 + #7,# = -22,300,000,000.000000
```
Furthermore, only ###/#num# will be replaced, allowing you to put things around
###/#num# to alter the final output:
Examples:
--------
```
2309.2093 + (#2#) = (2309.21)
2309.2093 + #2#** = 2309.21**
2309.2093 + ab#2#cd = ab2309.21cd
```
If you are doing exact substitution, then you can use characters:
Examples:
---------
`abc + ### = abc`
................................
Intentionally blank cells:
................................
If you would like to display a blank cell, you can use "---":
Examples:
---------
```
--- + ### = ---
--- + #3# = ---
```
######################
# Example Combinations
# Of input + template
######################
Example 1 (Simple)
----------
```
Input: <tab:Test>
1 2 3
2 1 3
3 1 2
Template: `<tab:Test> ` (pretend this is what you see in LyX)
### ### ###
### ### ###
### ### ###
Result:<tab:Test>
1 2 3
2 1 3
3 1 2
```
Example 2 (More Complicated)
----------
```
Input: <tab:Test>
1 . 3
2e-5 1 3.023
. -1 2 3
Template: <tab:Test> (pretend this is what you see in LyX)
(###) 2 ###
#3# ### #1#
NA ### ### ###
Result:<tab:Test>
(1) 2 3
0.000 1 3.0
NA -1 2 3
```
===================
====Important======
===================
By design, missings in input table and "missings" in template do not have to
line up.
Example 3 (LyX)
----------
```
Input: <tab:Test>
1 . 3
2e-5 . 3.023
. -1 2
Template: <tab:Test>
### ### abc
abc #2# #3#
NA ### ###
Result:<tab:Test>
1 3 abc
abc 0.00 3.023
NA -1 2
Recall that to the program, the above input table is no different from:
1 3
2e-5 3.023
-1 2
```
It doesn't "know" where the numbers should be placed within a row, only what
the next number to place should be.
Similarly:
Example 4 (LyX)
----------
```
Input: <tab:Test>
1 1 2
1 1 3
2 -1 2
Template: <tab:Test>
### ### ###
abc abc abc
### #2# #3#
### ### ###
Result:<tab:Test>
1 1 2
abc abc abc
1 1.00 3.000
2 -1 2
```
If a row in the template has no substitutions, then it's not really a row from
the program's point of view.
######################
# Error Logging
######################
If an error occurs during the call to tablefill, it will be displayed in the
command window. When make.py finishes, the user will be able to scroll up
through the output and examine any error messages. Error messages, which
include a description of the error type and a traceback to the line of code
where the error occured, can also be retuned as a string object using the
following syntax:
exitmessage = tablefill( input = 'input_file(s)', template = 'template_file',
output = 'output_file' )
Lines can then be added to make.py to output this string to a log file using
standard Python and built in gslab_make commands.
######################
# Common Errors
######################
Common mistakes which can lead to errors include:
- Mismatch between the length of the LyX table and the corresponding text table.
If the LyX table has more entries to be filled than the text table has entries to
fill from, this will cause an error and the table will not be filled.
- Use of numerical tags (e.g. #1#) to fill non-numerical data. This will cause
an error. Non-numerical data can only be filled using "###", as it does not make
sense to round or truncate this data.
- Multiple table objects in the same float. Each table float in the template LyX
file can only contain one table object. If a float contains a second table object,
this table will not be filled.
######################
# Boldfacing entries
######################
It is straightforward to develop functions that conditionally write entries of
tables in boldface; functions may do so by inserting '\series bold' in the lines
of the filled LyX file immeadiately before phrases that the user wishes to make bold.
"""
try:
inputs = convert_to_list(inputs, 'file')
inputs = [norm_path(file) for file in inputs]
content = [_parse_content(file, null) for file in inputs]
tables = {tag: data for (tag, data) in content}
if (len(content) != len(tables)):
raise_from(CritError(messages.crit_error_duplicate_tables), None)
doc = _insert_tables(template, tables, null)
with io.open(output, 'w', encoding='utf-8') as f:
f.write(doc)
except:
error_message = 'Error with `tablefill`. Traceback can be found below.'
error_message = format_message(error_message)
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def log_files_in_output(paths,
depth = float('inf')):
""".. Log files in output directory.
Logs the following information for all files contained in directory ``output_dir``.
- File name (in file ``output_statslog``)
- Last modified (in file ``output_statslog``)
- File size (in file ``output_statslog``)
- File head (in file ``output_headslog``, optional)
When walking through directory ``output_dir``, float ``depth`` determines level of depth to walk.
Status messages are appended to file ``makelog``.
Include additional output directories to walk through
(typically directories that you wish to keep local) in directory list ``output_local_dir``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
depth : float, optional
Level of depth when walking through output directory. Defaults to infinite.
Path Keys
---------
output_dir : str
Path of output directory.
output_local_dir : str, list, optional
Path or list of paths of local output directories. Defaults to ``[]`` (i.e., none).
output_statslog : str
Path to write output statistics log.
output_headslog : str, optional
Path to write output headers log.
makelog : str
Path of makelog.
Returns
-------
None
Example
-------
The following code will log information for all files contained in
only the first level of ``paths['output_dir']``.
Therefore, files contained in subdirectories will be ignored.
.. code-block:: python
log_files_in_outputs(paths, depth = 1)
The following code will log information for any file in ``paths['output_dir']``,
regardless of level of subdirectory.
.. code-block :: python
log_files_in_outputs(paths, depth = float('inf'))
"""
try:
output_dir = get_path(paths, 'output_dir')
output_local_dir = get_path(paths, 'output_local_dir', throw_error = False)
output_statslog = get_path(paths, 'output_statslog')
output_headslog = get_path(paths, 'output_headslog', throw_error = False)
if output_local_dir:
output_local_dir = convert_to_list(output_local_dir, 'dir')
else:
output_local_dir = []
output_files = glob_recursive(output_dir, depth)
output_local_files = [f for dir_path in output_local_dir for f in glob_recursive(dir_path, depth)]
output_files = set(output_files + output_local_files)
if output_statslog:
output_statslog = norm_path(output_statslog)
_write_stats_log(output_statslog, output_files)
if output_headslog:
output_headslog = norm_path(output_headslog)
_write_heads_log(output_headslog, output_files)
message = 'Output logs successfully written!'
write_to_makelog(paths, message)
print(colored(message, metadata.color_success))
except:
error_message = 'Error with `log_files_in_output`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths, error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def run_jupyter(paths, program, timeout=None, kernel_name=''):
""".. Run Jupyter notebook using system command.
Runs notebook ``program`` using Python API, with notebook specified
in the form of ``notebook.ipynb``.
Status messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
program : str
Path of script to run.
Path Keys
---------
makelog : str
Path of makelog.
Note
----
We recommend leaving all other parameters to their defaults.
Other Parameters
----------------
timeout : int, optional
Time to wait (in seconds) to finish executing a cell before raising exception.
Defaults to no timeout.
kernel_name : str, optional
Name of kernel to use for execution
(e.g., ``python2`` for standard Python 2 kernel, ``python3`` for standard Python 3 kernel).
Defaults to ``''`` (i.e., kernel specified in notebook).
Returns
-------
None
Example
-------
.. code-block:: python
run_jupyter(paths, program = 'notebook.ipynb')
"""
try:
program = norm_path(program)
with open(program) as f:
message = 'Processing notebook: `%s`' % program
write_to_makelog(paths, message)
print(colored(message, 'cyan'))
if not kernel_name:
kernel_name = 'python%s' % sys.version_info[0]
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name)
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {'metadata': {'path': '.'}})
with open(program, 'wt') as f:
nbformat.write(nb, f)
except:
error_message = 'Error with `run_jupyter`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
def write_source_logs(paths, source_map, depth=float('inf')):
""".. Write source logs.
Logs the following information for sources contained in list ``source_map``
(returned by :ref:`sourcing functions<sourcing functions>`).
- Mapping of symlinks/copies to sources (in file ``source_maplog``)
- Details on files contained in sources:
- File name (in file ``source_statslog``)
- Last modified (in file ``source_statslog``)
- File size (in file ``source_statslog``)
- File head (in file ``source_headlog``, optional)
When walking through sources, float ``depth`` determines level of depth to walk.
Status messages are appended to file ``makelog``.
Parameters
----------
paths : dict
Dictionary of paths. Dictionary should contain values for all keys listed below.
source_map : list
Mapping of symlinks/copies (destination) to sources (returned by :ref:`sourcing functions<sourcing functions>`).
depth : float, optional
Level of depth when walking through source directories. Defaults to infinite.
Path Keys
---------
source_statslog : str
Path to write source statistics log.
source_headslog : str, optional
Path to write source headers log.
source_maplog : str
Path to write source map log.
makelog : str
Path of makelog.
Returns
-------
None
Example
-------
The following code will log information for all files listed in ``source_map``.
Therefore, files contained in directories listed in ``source_map`` will be ignored.
.. code-block:: python
write_source_logs(paths, depth = 1)
The following code will log information for all files listed in ``source_map``
and any file in all directories listed in ``source_map``, regardless of level of subdirectory.
.. code-block :: python
write_source_logs(paths, depth = float('inf'))
"""
try:
source_statslog = get_path(paths, 'source_statslog')
source_headslog = get_path(paths, 'source_headslog', throw_error=False)
source_maplog = get_path(paths, 'source_maplog')
source_list = [source for source, destination in source_map]
source_list = [glob_recursive(source, depth) for source in source_list]
source_files = [f for source in source_list for f in source]
source_files = set(source_files)
# ACTION: DECIDE WHETHER TO ALLOW FOR RAW DIRECTORY
raw_dir = get_path(paths, 'raw_dir', throw_error=False)
if raw_dir:
raw_files = glob_recursive(raw_dir)
source_files = set(source_files + raw_files)
if source_statslog:
source_statslog = norm_path(source_statslog)
_write_stats_log(source_statslog, source_files)
if source_headslog:
source_headslog = norm_path(source_headslog)
_write_heads_log(source_headslog, source_files)
if source_maplog:
source_maplog = norm_path(source_maplog)
_write_source_maplog(source_maplog, source_map)
message = 'Source logs successfully written!'
write_to_makelog(paths, message)
print(colored(message, metadata.color_success))
except:
error_message = 'Error with `write_source_logs`. Traceback can be found below.'
error_message = format_message(error_message)
write_to_makelog(paths,
error_message + '\n\n' + traceback.format_exc())
raise_from(ColoredError(error_message, traceback.format_exc()), None)
