def build_python(target, source, env):
'''Build SCons targets using a Python script
This function executes a Python script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Python script that the builder is intended to execute.
'''
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'python')
log_file = target_dir + '/sconscript.log'
cl_arg = misc.command_line_args(env)
os.system('python %s %s > %s' % (source_file, cl_arg, log_file))
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_lyx(target, source, env):
'''Compile a pdf from a LyX file
This function is a SCons builder that compiles a .lyx file
as a pdf and places it at the path specified by target.
Parameters
----------
target: string or list
The target of the SCons command. This should be the path
of the pdf that the builder is instructed to compile.
source: string or list
The source of the SCons command. This should
be the .lyx file that the function will compile as a PDF.
'''
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'lyx')
newpdf = source_file.replace('.lyx', '.pdf')
log_file = target_dir + '/sconscript.log'
os.system('lyx -e pdf2 %s > %s' % (source_file, log_file))
shutil.move(newpdf, target_file)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_tables(target, source, env):
'''Build a SCons target by filling a table
This function uses the tablefill function from gslab_fill to produced a
filled table from (i) an empty table in a LyX file and (ii) text files
containing data to be used in filling the table.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the LyX file specifying the table format. The subsequent
sources should be the text files containing the data with which the
tables are to be filled.
'''
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
misc.check_code_extension(str(target[0]), 'lyx')
tablefill(input = ' '.join([str(a) for a in source[1:len(source)]]),
template = str(source[0]),
output = str(target[0]))
return None
def build_lyx(target, source, env):
'''Compile a pdf from a LyX file
This function is a SCons builder that compiles a .lyx file
as a pdf and places it at the path specified by target.
Parameters
----------
target: string or list
The target of the SCons command. This should be the path
of the pdf that the builder is instructed to compile.
source: string or list
The source of the SCons command. This should
be the .lyx file that the function will compile as a PDF.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
misc.check_code_extension(source_file, '.lyx')
# Set up target file and log file
newpdf = source_file[:-4] + '.pdf'
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'lyx')
newpdf = source_file.replace('.lyx', '.pdf')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# System call
try:
command = 'lyx -e pdf2 %s > %s' % (source_file, log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
# Move rendered pdf to the target
shutil.move(newpdf, target_file)
except subprocess.CalledProcessError:
message = misc.command_error_msg("lyx", command)
raise ExecCallError(message)
# Close log
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def test_make_list_if_string(self):
self.assertEqual(misc.make_list_if_string(['test', 'test']),
['test', 'test'])
self.assertEqual(misc.make_list_if_string('test'), ['test'])
self.assertEqual(misc.make_list_if_string(['test']), ['test'])
# We expect the function to raise Type Errors when it receives
# inputs that are not strings or lists
with self.assertRaises(TypeError):
self.assertEqual(misc.make_list_if_string(1), 1)
with self.assertRaises(TypeError):
self.assertEqual(misc.make_list_if_string(None), None)
def build_stata(target, source, env):
'''Build targets with a Stata command
This function executes a Stata script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Stata .do script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
start_time = misc.current_time()
# Set up source file and the original location of the log
source_file = str(source[0])
misc.check_code_extension(source_file, '.do')
loc_log = os.path.basename(source_file).replace('.do', '.log')
# Set up log file destination
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# Set up command line arguments
cl_arg = misc.command_line_args(env)
executable = misc.get_stata_executable(env)
command_skeleton = misc.get_stata_command(executable)
try:
command = command_skeleton % (source_file, cl_arg)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError:
message = misc.command_error_msg("Stata", command)
raise ExecCallError(message)
shutil.move(loc_log, log_file)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_r(target, source, env):
'''Build SCons targets using an R script
This function executes an R script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the R script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'r')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
cl_arg = misc.command_line_args(env)
if cl_arg != '':
if misc.is_unix(): # R has platform-specific cl_arg syntax
cl_arg = "'--args %s'" % cl_arg
else:
cl_arg = "\"--args %s\"" % cl_arg
# System call
try:
command = 'R CMD BATCH --no-save %s %s %s' % (cl_arg, source_file,
log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError:
message = misc.command_error_msg("R", command)
raise ExecCallError(message)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def collect_builder_logs(parent_dir, excluded_dirs = []):
''' Recursively return dictionary of files named sconscript*.log
in parent_dir and nested directories.
Also return timestamp from those sconscript.log
(snippet from SO 3964681)
excluded_dirs (str or list of str):
list of directories to be excluded from the search
'''
builder_log_collect = {}
# Store paths to logs in a list, found from platform-specific command line tool
rel_parent_dir = os.path.relpath(parent_dir)
log_name = '*sconscript*.log'
excluded_dirs = misc.make_list_if_string(excluded_dirs)
log_paths = misc.finder(rel_parent_dir, log_name, excluded_dirs)
# Read the file at each path to a log and store output complete-time in a dict at filename
for log_path in log_paths:
with open(log_path, 'rU') as f:
try:
s = f.readlines()[1] # line 0 = log start time, line 1 = log end time
except IndexError:
s = ''
s = s[s.find('{') + 1: s.find('}')] # find {} time identifier
try:
builder_log_end_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
except ValueError: # if the code breaks, there's no time identifier
beginning_of_time = datetime.min
builder_log_end_time = beginning_of_time
builder_log_collect[log_path] = builder_log_end_time
return builder_log_collect
def build_python(target, source, env):
'''Build SCons targets using a Python script
This function executes a Python script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Python script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, '.py')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
cl_arg = misc.command_line_args(env)
# System call
try:
command = 'python %s %s > %s' % (source_file, cl_arg, log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as ex:
message = misc.command_error_msg("Python", command)
raise ExecCallError('%s\n%s' % (message, ex.output))
# Close log
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def __init__(self,
target,
source,
env,
name='GSLab Builder',
valid_extensions=[],
exec_opts=''):
'''
Fill builder with information about build step.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
name: string
Name of builder-type. Use to refer to builder in error messages and env.
valid_extensions: iterable
Valid (case-insensitive) extensions for first element of source list.
default_exec: string
Executable used to execute first element of source list.
Override by passing value through env.
exec_opts: string
Options used to execute first element of source list.
'''
# Store keyword args
self.name = name
self.valid_extensions = valid_extensions
self.exec_opts = exec_opts
# Build system call and store components
self.source_file = str(misc.make_list_if_string(source)[0])
self.target = [str(t) for t in misc.make_list_if_string(target)]
self.target_dir = misc.get_directory(self.target[0])
if 'executable_names' not in env:
env['executable_names'] = {}
self.executable = misc.get_executable(name, env['executable_names'])
self.env = env
self.add_command_line_arg()
self.add_log_file()
self.add_call_args()
self.system_call = '%s %s %s' % (self.executable, self.exec_opts,
self.call_args)
return None
def build_anything(target, source, action, env, warning=True, **kw):
'''
Anything builder-generator. The generator will create a custom builder
that runs `action` and add it as a SCons node, similar to the native env.Command.
Using gslab_scons.build_anything will utilize our logging mechanism
and error catching similar to our other builders.
`
Parameters
target: string or list
The target(s) of the SCons command. The log ends up in the
directory of the first specified target.
source: string or list
The source(s) of the SCons command.
action: string
The code to be run by the generated builder.
env: SCons construction environment, see SCons user guide 7.2.
You *** MUST *** manually pass `env = env` when calling this in SConscript,
since this is not a Scons.env method like env.Command.
Special parameters that can be added when using the builder
log_ext: string
Instead of logging to `sconscript.log` in the target dir,
the builder will log to `sconscript_<log_ext>.log`.
origin_log_file: string
Sometimes, your command may produce a log file in an undesirable location.
Specifying the that location in this argument leads the builder to append
the content of origin_log_file to log_file and delete origin_log_file.
The builder will crash if this file doesn't exist at the end of the command.
warning: Boolean
Turns off warnings if warning = False.
'''
import SCons.Builder
builder_attributes = {'name': 'Anything Builder'}
target = [t for t in misc.make_list_if_string(target) if t]
source = [s for s in misc.make_list_if_string(source) if s]
local_env = env.Clone()
for k, v in kw.items():
local_env[k] = v
builder = AnythingBuilder(target, source, action, local_env, warning,
**builder_attributes)
bkw = {
'action': builder.build_anything,
'target_factory': local_env.fs.Entry,
'source_factory': local_env.fs.Entry,
}
bld = SCons.Builder.Builder(**bkw)
return bld(local_env, target, source)
def __init__(self, target, source, action, env, warning=True, name=''):
'''
'''
target = [self.to_str(t) for t in misc.make_list_if_string(target)]
source = [self.to_str(s) for s in misc.make_list_if_string(source)]
self.action = action
super(AnythingBuilder, self).__init__(target, source, env, name=name)
try:
origin_log_file = env['origin_log_file']
except KeyError:
origin_log_file = None
self.origin_log_file = origin_log_file
if '>' in action and warning == True:
warning_message = '\nThere is a redirection operator > in ' \
'your prescribed action key.\n' \
'The Anything Builder\'s logging mechanism '\
'may not work as intended.'
warnings.warn(warning_message)
return None
def add_source_file(self, source):
'''
Add source file to execute as the first element of source.
If source is an empty list, then the source file is ''.
'''
if bool(source):
sources = misc.make_list_if_string(source)
source_file = str(sources[0])
else:
source_file = ''
self.source_file = os.path.normpath("%s" % source_file)
return None
def test_make_list_if_string(self):
self.assertEqual(misc.make_list_if_string(['test', 'test']),
['test', 'test'])
self.assertEqual(misc.make_list_if_string('test'), ['test'])
self.assertEqual(misc.make_list_if_string(['test']), ['test'])
# We expect objects that are neither strings nor lists to be
# returned without being manipulated.
self.assertEqual(misc.make_list_if_string(1), 1)
self.assertEqual(misc.make_list_if_string(TypeError), TypeError)
self.assertEqual(misc.make_list_if_string(False), False)
self.assertEqual(misc.make_list_if_string(None), None)
def check_code_extension(self):
'''
Raise an exception if the extension in executing script
does not mach extension in valid_extensions attribute.
'''
extensions = misc.make_list_if_string(self.valid_extensions)
if extensions == []:
return None
matches = [True for extension in extensions
if self.source_file.lower().endswith("%s" % extension)]
if not matches:
message = 'First argument, %s, must be a file of type %s.' % (self.source_file, extensions)
raise BadExtensionError(message)
return None
def add_command_line_arg(self):
'''
Store arguments to pass to the executing script on the command line.
Return the content of env['CL_ARG'] as a string with spaces separating entries.
If env['CL_ARG'] doesn't exist, return an empty string.
'''
try:
cl_arg = self.env['CL_ARG']
except KeyError:
cl_arg = ''
try:
cl_arg = misc.make_list_if_string(cl_arg)
cl_arg = ' '.join([str(s) for s in cl_arg])
except TypeError:
cl_arg = str(cl_arg)
self.cl_arg = "%s" % cl_arg
return None
def build_matlab(target, source, env):
'''Build targets with a MATLAB command
This function executes a MATLAB function to build objects
specified by target using the objects specified by source.
It requires MATLAB to be callable from the command line
via `matlab`.
Accessing command line arguments from within matlab is
possible via the `command_line_arg = getenv('CL_ARG')`.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the MATLAB .M script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
if misc.is_unix():
options = '-nosplash -nodesktop'
elif sys.platform == 'win32':
options = '-nosplash -minimize -wait'
else:
raise PrerequisiteError("Unsupported OS")
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, '.m')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# Set up command line arguments
cl_arg = misc.command_line_args(env)
os.environ['CL_ARG'] = cl_arg
# Run MATLAB on source file
shutil.copy(source_file, 'source.m')
try:
command = 'matlab %s -r source > %s' % (options, log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError:
message = misc.command_error_msg("Matlab", command)
raise ExecCallError(message)
os.remove('source.m')
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_stata(target, source, env):
'''Build targets with a Stata command
This function executes a Stata script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Stata .do script that the builder is intended to
execute.
Note: the user can specify a flavour by typing `scons sf=StataMP`
(By default, SCons will try to find each flavour).
'''
cl_arg = misc.command_line_args(env)
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'stata')
log_file = target_dir + '/sconscript.log'
loc_log = os.path.basename(source_file).replace('.do', '.log')
user_flavor = env['user_flavor']
if user_flavor is not None:
if misc.is_unix():
command = misc.stata_command_unix(user_flavor, cl_arg)
elif sys.platform == 'win32':
command = misc.stata_command_win(user_flavor, cl_arg)
else:
flavors = ['stata-mp', 'stata-se', 'stata']
if misc.is_unix():
for flavor in flavors:
if misc.is_in_path(flavor):
command = misc.stata_command_unix(flavor, cl_arg)
break
elif sys.platform == 'win32':
try:
key_exist = os.environ['STATAEXE'] is not None
command = misc.stata_command_win("%%STATAEXE%%")
except KeyError:
flavors = [(f.replace('-', '') + '.exe') for f in flavors]
if misc.is_64_windows():
flavors = [f.replace('.exe', '-64.exe') for f in flavors]
for flavor in flavors:
if misc.is_in_path(flavor):
command = misc.stata_command_win(flavor, cl_arg)
break
try:
subprocess.check_output(command % source_file,
stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError:
raise BadExecutableError('Could not find executable.')
shutil.move(loc_log, log_file)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_tables(target, source, env):
'''Build a SCons target by filling a table
This function uses the tablefill function from gslab_fill to produced a
filled table from (i) an empty table in a LyX/Tex file and (ii) text files
containing data to be used in filling the table.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the LyX/Tex file specifying the table format. The subsequent
sources should be the text files containing the data with which the
tables are to be filled.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
start_time = misc.current_time()
# Set up source file (table format)
source_file = str(source[0])
misc.check_code_extension(source_file, ['.lyx', '.tex'])
# Set up input string (list of data tables)
input_string = ' '.join([str(i) for i in source[1:]])
# Set up target file (filled table)
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
misc.check_code_extension(target_file, ['.lyx', '.tex'])
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# Command call
output = tablefill(input=input_string,
template=source_file,
output=target_file)
with open(log_file, 'wb') as f:
f.write(output)
f.write("\n\n")
# Close log
if "traceback" in str.lower(output): # if tablefill.py returns an error
command = """tablefill(input = %s,
template = %s,
output = %s)""" % (input_string, source_file,
target_file)
message = misc.command_error_msg("tablefill.py", command)
raise ExecCallError(message)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
