def get_externals(externals_file,
external_dir='@DEFAULTVALUE@',
makelog='@DEFAULTVALUE@',
quiet=False):
'''Fetch external files
Description:
This function interprets a formatted text document listing files
to be exported via SVN or a system copy command.
Syntax:
get_externals(externals_file [, externals_dir [, makelog [, quiet]]])
Usage:
The `externals_file` argument should be the path of a tab-delimited text
file containing information on the external files that the function call
should retrieve. This file needs to rows of numbers or characters, delimited
by either tabs or 4 spaces,one for each file to be exported via svn.
The proper format is: rev dir file outdir outfile notes
### Column descriptions:
* rev
* Revision number of the file/directory in integer format.
If left blank along with directory column, get_externals.py will
read the last specified revision number. If copying from a shared
drive rather than the repository, list revision number as COPY.
* dir
* Directory of the file/directory requested. As described above,
%xxx% placemarkers are substituted in from predefined values in
metadata.py. If left blank along with revision column,
get_externals.py will read the last specified directory.
* file
* Name of the file requested. If entire directory is required, leave
column as a single *. If a file name wildcard is required place
single * within filename. get_externals.py will attempt to screen
out bad file names. Cannot be left blank.
* outdir
* Desired output directory of the exported file/directory.
Typically of the form ./subdir/. If left blank, will be
filled with the first level of the externals relative path.
* outfile
* Desired output name of the exported file/directory. If left as
double quotes, indicates that it should have the same name.
Adding a directory name that is different from the default """"
will place this subdirectory within the outdir. Additionally,
get_externals can assign a prefix tag to exported file collections,
either through a folder export, or a wildcard call; it does so
when the outfile column contains text of the pattern '[prefix]*',
where the prefix [prefix] will be attached to exported files.
* notes
* Optional column with notes on the export. get_externals.py ignores this,
but logs it.
Example of externals.txt:
```
rev dir file outdir outfile notes
2 %svn%/directory/ * ./destination_directory/ """"
COPY %svn%/other_directory/ my_file.txt . """"
```
The destination directory is specified by an optional second
parameter whose default value is "../external". The log file produced by
get_externals is automatically added to an optional third parameter
whose default value is '../output/make.log'.
The fourth argument, quiet, is by default False. Setting this argument to
True suppresses standard output and errors from SVN.
'''
try:
LOGFILE = prelim.start_logging(metadata.settings['externalslog_file'],
'get_externals.py')
makelog, externals, last_dir, last_rev = \
prelim.externals_preliminaries(makelog, externals_file, LOGFILE)
for line in externals:
try:
directive = SystemDirective(line, LOGFILE, last_dir, last_rev)
directive.error_check()
directive.clean(external_dir)
directive.issue_sys_command(quiet)
# Save rev/dir for next line
last_dir = directive.dir
last_rev = directive.rev
except:
prelim.print_error(LOGFILE)
prelim.end_logging(LOGFILE, makelog, 'get_externals.py')
except Exception as errmsg:
print "Error with get_external: \n", errmsg
def make_links(links_files,
links_dir='@DEFAULTVALUE@',
makelog='@DEFAULTVALUE@',
quiet=False):
'''Import the make_links function from the make_links.py module.
Description:
This function parses and interprets links files and uses that input to create symlinks
to large data files (symlinks allow shortcuts to be made locally so that programs will
act as if the files are stored locally, but no local hard drive space is taken up)
Syntax:
make_links(links_files [, links_dir [, makelog [, quiet]]])
Parameters:
- links_files: A string containing the name of a valid links file, a string containing
multiple names of valid links files (delimited by spaces), or a Python list of
strings in which each element is a valid links file. Each string may also contain one
wildcard (as an asterisk). For example:
`make_links('./links.txt')`, or
`make_links('../external/links1.txt ../external/links2.txt')`, or
`make_links(['../external/links1.txt', '../external/links2.txt'])`, or
`make_links('../external/links*.txt')`.
To get every file in a directory, simply end the string with a wild card:
`make_links('../external/links_files/*')`
There is no default argument for this parameter; links_files must be included in
every call to make_links().
- links_dir: A string containing the name of the local directory into which the symlinks
will be created. The default argument for this parameter is '../external_links/'.
- makelog: A string containing the name of the makelog that will store make_links()
output. If makelog is an empty string (i.e. ''), the default log file will be
unchanged. The default argument for this parameter is '../output/make.log'.
- quiet: A boolean. Setting this argument to True suppresses standard output and errors
from svn. The default argument for this parameter is False.
Caution:
Take care when dealing with symlinks. Most programs will treat the links as if they are
the file being linked to. For instance, if ../data_links/some_data.dta is a link to
\external_source\some_data.dta and Stata opens ../data_links/some_data.dta, edits it,
and saves it to the same location, the original file (\external_source\some_data.dta)
will be changed. In such a case, it would probably be preferable to save a local copy of
the data and make changes to that file.
'''
try:
if makelog == '@DEFAULTVALUE@':
makelog = metadata.settings['makelog_file']
# Run preliminaries
LOGFILE = start_logging(metadata.settings['linkslog_file'],
'make_links.py')
list = LinksList(links_files, links_dir)
if os.path.exists(list.links_dir):
remove_dir(list.links_dir)
os.makedirs(list.links_dir)
list.issue_sys_command(LOGFILE, quiet)
end_logging(LOGFILE, makelog, 'make_links.py')
except Exception as errmsg:
print "Error with make_links: \n", errmsg
def get_externals_github(externals_file,
external_dir='@DEFAULTVALUE@',
makelog='@DEFAULTVALUE@',
quiet=False):
'''Fetch external files from GitHub
Description:
This function retrieves files from GitHub as specified by a formatted text
document indicated as an argument.
get_externals_github is only intended to download assets from releases.
It cannot download the release source code or other assets not contained
within a release.
Usage:
- This function's usage largely follows that of get_externals(). Instead of
rading
- get_externals_github is only intended to download assets from releases.
It cannot download the release source code or other assets not contained
within a release.
File format of a `externals_file`:
This file needs to rows of numbers or characters, delimited by either tabs or
4 spaces, one for each file to be exported via GitHub.
The proper column format is: url outdir outfile notes
Column descriptions:
* url
* Download url for the file in a specific GitHub release. The url given
should be the complete url.
* outdir
* Desired output directory of the exported file/directory. Typically of the form
./subdir/. If left blank, will be filled with the first level of the externals
relative path.
* outfile
* Desired output name of the exported file/directory. If left as double quotes,
indicates that it should have the same name as the asset name in GitHub.
* notes
* Optional column with notes on the export. get_externals_github.py ignores this,
but logs it.
Example of externals_github.txt:
```
url outdir outfile notes
https://github.com/TestUser/TestRepo/releases/download/v1.0/document.pdf ./ """"
https://github.com/TestUser/TestRepo/releases/download/AlternativeVersion/other_document.pdf ./ """"
```
'''
try:
# Request Token
token = getpass.getpass(
"\nEnter a valid GitHub token and then press enter: ")
LOGFILE = prelim.start_logging(metadata.settings['githublog_file'],
'get_externals_github.py')
makelog, externals, last_dir, last_rev = \
prelim.externals_preliminaries(makelog, externals_file, LOGFILE)
for line in externals:
try:
directive = SystemDirective(line,
LOGFILE,
last_dir,
last_rev,
token=token)
directive.error_check()
directive.clean(external_dir)
directive.issue_sys_command(quiet)
except:
prelim.print_error(LOGFILE)
prelim.end_logging(LOGFILE, makelog, 'get_externals_github.py')
except Exception as errmsg:
print "Error with get_externals_github: \n", errmsg