def list_directory(top, makelog='@DEFAULTVALUE@'):
"""List directories
This function lists all non-hidden sub-directories of the directory
specified by `top`, a path, and their content from the top down.
It writes their names, modified times, and sizes in bytes to the
log file specified by the path `makelog`.
"""
# metadata.settings should not be part of argument defaults so that they can be
# overwritten by make_log.set_option
if makelog == '@DEFAULTVALUE@':
makelog = metadata.settings['makelog_file']
print "\nList all files in directory", top
# To print numbers (file sizes) with thousand separator
locale.setlocale(locale.LC_ALL, '')
makelog = re.sub('\\\\', '/', makelog)
try:
LOGFILE = open(makelog, 'ab')
except Exception as errmsg:
print errmsg
raise CritError(messages.crit_error_log % makelog)
print >> LOGFILE, '\n'
print >> LOGFILE, 'List of all files in sub-directories in', top
try:
if os.path.isdir(top):
for root, dirs, files in os.walk(top, topdown=True):
# Ignore non-hidden sub-directories
dirs_to_keep = []
for dirname in dirs:
if not dirname.startswith('.'):
dirs_to_keep.append(dirname)
dirs[:] = dirs_to_keep
# Print out the sub-directory and its time stamp
created = os.stat(root).st_mtime
asciiTime = time.asctime(time.localtime(created))
print >> LOGFILE, root
print >> LOGFILE, 'created/modified', asciiTime
# Print out all the files in the sub-directories
for name in files:
full_name = os.path.join(root, name)
created = os.path.getmtime(full_name)
size = os.path.getsize(full_name)
asciiTime = time.asctime(time.localtime(created))
print >> LOGFILE, '%50s' % name, '--- created/modified', asciiTime, \
'(', locale.format('%d', size, 1), 'bytes )'
except:
print_error(LOGFILE)
print >> LOGFILE, '\n'
LOGFILE.close()
def del_log(*args, **kwargs):
"""Delete log files
This function deletes each of the log files listed in "*arg" list.
Errors are printed to `makelog` log file.
- After we append the various log files to the common one using add_log, we may
want to delete the extra ones to clean up the output folder. Here is the command:
`del_log('../output/analysis.log', '../output/another_log.log')
- If variable 'makelog' is not defined in kwargs, errors from del_log will be printed
to the default makelog (usually '../output/make.log'). If we want to specify a
different log to which we print the errors:
`del_log('../output/analysis.log', '../output/another_log.log',
makelog = '../different.log')`
"""
if 'makelog' in kwargs.keys():
makelog = kwargs['makelog']
else:
makelog = metadata.settings['makelog_file']
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise CritError(messages.crit_error_nomakelog % makelog)
print "\nDelete log file(s)"
makelog = re.sub('\\\\', '/', makelog)
try:
LOGFILE = open(makelog, 'ab')
except Exception as errmsg:
raise CritError((messages.crit_error_log % makelog) + '\n' +
str(errmsg))
try:
for log in args:
log = re.sub('\\\\', '/', log)
if os.path.isfile(log):
os.remove(log)
else:
print >> LOGFILE, messages.note_nofile % log
except:
print_error(LOGFILE)
LOGFILE.close()
def add_log(*args, **kwargs):
"""Add log files in "*arg" list to "makelog" log file.
- If there are log files created inside scripts (for example
from a Perl or Python script), and we want to append their content
to the end of the default log file (usually '../output/make.log'), we
would use: `add_log('../output/analysis.log', '../output/another_log.log')`
- The number of log arguments can vary. If the log files don't actually exist,
errors will be printed to the common log file.
- If we want to append content of log files to a different log file than the
default, we would use:
`add_log('../output/analysis.log', '../output/another_log.log', makelog = '../different.log')`
"""
if 'makelog' in kwargs.keys():
makelog = kwargs['makelog']
else:
makelog = metadata.settings['makelog_file']
if not (metadata.makelog_started and os.path.isfile(makelog)):
raise CritError(messages.crit_error_nomakelog % makelog)
print "\nAdd log file(s) to: ", makelog
makelog = re.sub('\\\\', '/', makelog)
try:
LOGFILE = open(makelog, 'ab')
except Exception as errmsg:
raise CritError((messages.crit_error_log % makelog) + '\n' +
str(errmsg))
try:
for log in args:
log = re.sub('\\\\', '/', log)
if not os.path.isfile(log):
print >> LOGFILE, messages.note_nofile % log
else:
LOGFILE.write(open(log, 'rU').read())
except:
print_error(LOGFILE)
LOGFILE.close()
def check_manifest(manifestlog='@DEFAULTVALUE@',
output_dir='@DEFAULTVALUE@',
makelog='@DEFAULTVALUE@'):
"""
Produce an error if there are any .dta files in "output_dir" and all
non-hidden sub-directories that are not in the manifest file "manifestlog",
and produce a warning if there are .txt or .csv files not in the manifest
along with a list of these files. All log is printed to "makelog" log file.
"""
# metadata.settings should not be part of argument defaults so that they can be
# overwritten by make_log.set_option
if manifestlog == '@DEFAULTVALUE@':
manifestlog = metadata.settings['manifest_file']
if output_dir == '@DEFAULTVALUE@':
output_dir = metadata.settings['output_dir']
if makelog == '@DEFAULTVALUE@':
makelog = metadata.settings['makelog_file']
print "\nCheck manifest log file", manifestlog
# Open main log file
try:
LOGFILE = open(makelog, 'ab')
except Exception as errmsg:
print errmsg
raise CritError(messages.crit_error_log % makelog)
try:
# Open manifest log file
try:
MANIFESTFILE = open(manifestlog, 'rU')
except Exception as errmsg:
print errmsg
raise CritError(messages.crit_error_log % manifestlog)
manifest_lines = MANIFESTFILE.readlines()
MANIFESTFILE.close()
# Get file list
try:
file_list = []
for i in range(len(manifest_lines)):
if manifest_lines[i].startswith('File: '):
filepath = os.path.abspath(manifest_lines[i][6:].rstrip())
ext = os.path.splitext(filepath)[1]
if ext == '':
filepath = filepath + '.dta'
file_list.append(filepath)
except Exception as errmsg:
print errmsg
raise SyntaxError(messages.syn_error_manifest % manifestlog)
if not os.path.isdir(output_dir):
raise CritError(messages.crit_error_no_directory % (output_dir))
# Loop over all levels of sub-directories of output_dir
for root, dirs, files in os.walk(output_dir, topdown=True):
# Ignore non-hidden sub-directories
dirs_to_keep = []
for dirname in dirs:
if not dirname.startswith('.'):
dirs_to_keep.append(dirname)
dirs[:] = dirs_to_keep
# Check each file
for filename in files:
ext = os.path.splitext(filename)[1]
fullpath = os.path.abspath(os.path.join(root, filename))
# non-hidden .dta file: error
if (not filename.startswith('.')) and (ext == '.dta'):
print 'Checking: ', fullpath
if not (fullpath in file_list):
raise CritError(messages.crit_error_no_dta_file %
(filename, manifestlog))
# non-hidden .csv file: warning
if (not filename.startswith('.')) and (ext == '.csv'):
print 'Checking: ', fullpath
if not (fullpath in file_list):
print messages.note_no_csv_file % (filename,
manifestlog)
print >> LOGFILE, messages.note_no_csv_file % (
filename, manifestlog)
# non-hidden .txt file: warning
if (not filename.startswith('.')) and (ext == '.txt'):
print 'Checking: ', fullpath
if not (fullpath in file_list):
print messages.note_no_txt_file % (filename,
manifestlog)
print >> LOGFILE, messages.note_no_txt_file % (
filename, manifestlog)
except:
print_error(LOGFILE)
LOGFILE.close()
def get_externals(externals_file,
external_dir='@DEFAULTVALUE@',
makelog='@DEFAULTVALUE@',
quiet=False):
'''Fetch external files
Description:
This function interprets a formatted text document listing files
to be exported via SVN or a system copy command.
Syntax:
get_externals(externals_file [, externals_dir [, makelog [, quiet]]])
Usage:
The `externals_file` argument should be the path of a tab-delimited text
file containing information on the external files that the function call
should retrieve. This file needs to rows of numbers or characters, delimited
by either tabs or 4 spaces,one for each file to be exported via svn.
The proper format is: rev dir file outdir outfile notes
### Column descriptions:
* rev
* Revision number of the file/directory in integer format.
If left blank along with directory column, get_externals.py will
read the last specified revision number. If copying from a shared
drive rather than the repository, list revision number as COPY.
* dir
* Directory of the file/directory requested. As described above,
%xxx% placemarkers are substituted in from predefined values in
metadata.py. If left blank along with revision column,
get_externals.py will read the last specified directory.
* file
* Name of the file requested. If entire directory is required, leave
column as a single *. If a file name wildcard is required place
single * within filename. get_externals.py will attempt to screen
out bad file names. Cannot be left blank.
* outdir
* Desired output directory of the exported file/directory.
Typically of the form ./subdir/. If left blank, will be
filled with the first level of the externals relative path.
* outfile
* Desired output name of the exported file/directory. If left as
double quotes, indicates that it should have the same name.
Adding a directory name that is different from the default """"
will place this subdirectory within the outdir. Additionally,
get_externals can assign a prefix tag to exported file collections,
either through a folder export, or a wildcard call; it does so
when the outfile column contains text of the pattern '[prefix]*',
where the prefix [prefix] will be attached to exported files.
* notes
* Optional column with notes on the export. get_externals.py ignores this,
but logs it.
Example of externals.txt:
```
rev dir file outdir outfile notes
2 %svn%/directory/ * ./destination_directory/ """"
COPY %svn%/other_directory/ my_file.txt . """"
```
The destination directory is specified by an optional second
parameter whose default value is "../external". The log file produced by
get_externals is automatically added to an optional third parameter
whose default value is '../output/make.log'.
The fourth argument, quiet, is by default False. Setting this argument to
True suppresses standard output and errors from SVN.
'''
try:
LOGFILE = prelim.start_logging(metadata.settings['externalslog_file'],
'get_externals.py')
makelog, externals, last_dir, last_rev = \
prelim.externals_preliminaries(makelog, externals_file, LOGFILE)
for line in externals:
try:
directive = SystemDirective(line, LOGFILE, last_dir, last_rev)
directive.error_check()
directive.clean(external_dir)
directive.issue_sys_command(quiet)
# Save rev/dir for next line
last_dir = directive.dir
last_rev = directive.rev
except:
prelim.print_error(LOGFILE)
prelim.end_logging(LOGFILE, makelog, 'get_externals.py')
except Exception as errmsg:
print "Error with get_external: \n", errmsg
def get_externals_github(externals_file,
external_dir='@DEFAULTVALUE@',
makelog='@DEFAULTVALUE@',
quiet=False):
'''Fetch external files from GitHub
Description:
This function retrieves files from GitHub as specified by a formatted text
document indicated as an argument.
get_externals_github is only intended to download assets from releases.
It cannot download the release source code or other assets not contained
within a release.
Usage:
- This function's usage largely follows that of get_externals(). Instead of
rading
- get_externals_github is only intended to download assets from releases.
It cannot download the release source code or other assets not contained
within a release.
File format of a `externals_file`:
This file needs to rows of numbers or characters, delimited by either tabs or
4 spaces, one for each file to be exported via GitHub.
The proper column format is: url outdir outfile notes
Column descriptions:
* url
* Download url for the file in a specific GitHub release. The url given
should be the complete url.
* outdir
* Desired output directory of the exported file/directory. Typically of the form
./subdir/. If left blank, will be filled with the first level of the externals
relative path.
* outfile
* Desired output name of the exported file/directory. If left as double quotes,
indicates that it should have the same name as the asset name in GitHub.
* notes
* Optional column with notes on the export. get_externals_github.py ignores this,
but logs it.
Example of externals_github.txt:
```
url outdir outfile notes
https://github.com/TestUser/TestRepo/releases/download/v1.0/document.pdf ./ """"
https://github.com/TestUser/TestRepo/releases/download/AlternativeVersion/other_document.pdf ./ """"
```
'''
try:
# Request Token
token = getpass.getpass(
"\nEnter a valid GitHub token and then press enter: ")
LOGFILE = prelim.start_logging(metadata.settings['githublog_file'],
'get_externals_github.py')
makelog, externals, last_dir, last_rev = \
prelim.externals_preliminaries(makelog, externals_file, LOGFILE)
for line in externals:
try:
directive = SystemDirective(line,
LOGFILE,
last_dir,
last_rev,
token=token)
directive.error_check()
directive.clean(external_dir)
directive.issue_sys_command(quiet)
except:
prelim.print_error(LOGFILE)
prelim.end_logging(LOGFILE, makelog, 'get_externals_github.py')
except Exception as errmsg:
print "Error with get_externals_github: \n", errmsg