def build_lyx(target, source, env):
'''Compile a pdf from a LyX file
This function is a SCons builder that compiles a .lyx file
as a pdf and places it at the path specified by target.
Parameters
----------
target: string or list
The target of the SCons command. This should be the path
of the pdf that the builder is instructed to compile.
source: string or list
The source of the SCons command. This should
be the .lyx file that the function will compile as a PDF.
'''
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'lyx')
newpdf = source_file.replace('.lyx', '.pdf')
log_file = target_dir + '/sconscript.log'
os.system('lyx -e pdf2 %s > %s' % (source_file, log_file))
shutil.move(newpdf, target_file)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_python(target, source, env):
'''Build SCons targets using a Python script
This function executes a Python script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Python script that the builder is intended to execute.
'''
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'python')
log_file = target_dir + '/sconscript.log'
cl_arg = misc.command_line_args(env)
os.system('python %s %s > %s' % (source_file, cl_arg, log_file))
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_lyx(target, source, env):
'''Compile a pdf from a LyX file
This function is a SCons builder that compiles a .lyx file
as a pdf and places it at the path specified by target.
Parameters
----------
target: string or list
The target of the SCons command. This should be the path
of the pdf that the builder is instructed to compile.
source: string or list
The source of the SCons command. This should
be the .lyx file that the function will compile as a PDF.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
misc.check_code_extension(source_file, '.lyx')
# Set up target file and log file
newpdf = source_file[:-4] + '.pdf'
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'lyx')
newpdf = source_file.replace('.lyx', '.pdf')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# System call
try:
command = 'lyx -e pdf2 %s > %s' % (source_file, log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
# Move rendered pdf to the target
shutil.move(newpdf, target_file)
except subprocess.CalledProcessError:
message = misc.command_error_msg("lyx", command)
raise ExecCallError(message)
# Close log
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_stata(target, source, env):
'''Build targets with a Stata command
This function executes a Stata script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Stata .do script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
start_time = misc.current_time()
# Set up source file and the original location of the log
source_file = str(source[0])
misc.check_code_extension(source_file, '.do')
loc_log = os.path.basename(source_file).replace('.do', '.log')
# Set up log file destination
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# Set up command line arguments
cl_arg = misc.command_line_args(env)
executable = misc.get_stata_executable(env)
command_skeleton = misc.get_stata_command(executable)
try:
command = command_skeleton % (source_file, cl_arg)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError:
message = misc.command_error_msg("Stata", command)
raise ExecCallError(message)
shutil.move(loc_log, log_file)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_r(target, source, env):
'''Build SCons targets using an R script
This function executes an R script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the R script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'r')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
cl_arg = misc.command_line_args(env)
if cl_arg != '':
if misc.is_unix(): # R has platform-specific cl_arg syntax
cl_arg = "'--args %s'" % cl_arg
else:
cl_arg = "\"--args %s\"" % cl_arg
# System call
try:
command = 'R CMD BATCH --no-save %s %s %s' % (cl_arg, source_file,
log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError:
message = misc.command_error_msg("R", command)
raise ExecCallError(message)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def execute_system_call(self):
'''
Execute the system call attribute.
Log the execution.
Check that expected targets exist after execution.
'''
self.check_code_extension()
start_time = misc.current_time()
self.do_call()
self.check_targets()
end_time = misc.current_time()
self.timestamp_log(start_time, end_time)
return None
def build_python(target, source, env):
'''Build SCons targets using a Python script
This function executes a Python script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Python script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, '.py')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
cl_arg = misc.command_line_args(env)
# System call
try:
command = 'python %s %s > %s' % (source_file, cl_arg, log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as ex:
message = misc.command_error_msg("Python", command)
raise ExecCallError('%s\n%s' % (message, ex.output))
# Close log
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def test_current_time(self):
'''
Test that current_time() prints the correct current time
in the expected format. Here, the "correct current time"
is a fixed value set within the MockDateTime class.
'''
the_time = misc.current_time()
self.assertEqual('2017-01-20 15:02:18', the_time)
def end_log(cl_args_list=sys.argv,
log='sconstruct.log',
excluded_dirs=[],
release_dir='./release/'):
'''Complete the log of a build process.'''
if misc.is_scons_dry_run(cl_args_list=cl_args_list):
return None
end_message = "*** Build completed: {%s} ***\n \n \n" % misc.current_time()
with open(log, "a") as f:
f.write(end_message)
# scan sconstruct.log for start time
with open(log, "rU") as f:
s = f.readline()
s = s[s.find('{') + 1:s.find('}')]
start_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
# gather all sconscript logs
parent_dir = os.getcwd()
builder_logs = collect_builder_logs(parent_dir,
excluded_dirs=excluded_dirs)
# keep only builder logs from this run OR is broken (value == beginning_of_time)
beginning_of_time = datetime.min # to catch broken logs (see collect_builder_logs)
this_run_dict = {
key: value
for key, value in builder_logs.items()
if (value > start_time) or value == beginning_of_time
}
this_run_list = sorted(this_run_dict, key=this_run_dict.get, reverse=True)
with open(log, "a") as sconstruct:
for f in this_run_list:
with open(f, 'rU') as sconscript:
if this_run_dict[f] == beginning_of_time:
warning_string = "*** Warning!!! The log below does not have timestamps," + \
" the Sconscript may not have finished.\n"
sconstruct.write(warning_string)
sconstruct.write(f + '\n')
sconstruct.write(sconscript.read())
# move top level logs to /release/ directory.
if not os.path.exists(release_dir):
os.makedirs(release_dir)
for file in glob.glob("*.log"):
shutil.move('./' + file, release_dir + file)
return None
def start_log(mode, vers, log='sconstruct.log'):
'''Begins logging a build process'''
if not (mode in ['develop', 'cache']):
raise Exception("Error: %s is not a defined mode" % mode)
start_message = "*** New build: {%s} ***\n" % misc.current_time()
with open(log, "w") as f:
f.write(start_message)
if misc.is_unix():
sys.stdout = os.popen('tee -a %s' % log, 'wb')
elif sys.platform == 'win32':
sys.stdout = open(log, 'ab')
sys.stderr = sys.stdout
return None
def end_log(log='sconstruct.log'):
'''Complete the log of a build process.'''
end_message = "*** Build completed: {%s} ***\n \n \n" % misc.current_time()
with open(log, "a") as f:
f.write(end_message)
# scan sconstruct.log for start time
with open('sconstruct.log', "rU") as f:
s = f.readline()
s = s[s.find('{') + 1:s.find('}')]
start_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
# gather all sconscript logs
parent_dir = os.getcwd()
builder_logs = collect_builder_logs(parent_dir)
# keep only builder logs from this run OR is broken (value == beginning_of_time)
beginning_of_time = datetime.min # to catch broken logs (see collect_builder_logs)
this_run_dict = {
key: value
for key, value in builder_logs.items()
if (value > start_time) or value == beginning_of_time
}
this_run_list = sorted(this_run_dict, key=this_run_dict.get, reverse=True)
with open('sconstruct.log', "a") as sconstruct:
for f in this_run_list:
with open(f, 'rU') as sconscript:
if this_run_dict[f] == beginning_of_time:
warning_string = "*** Warning!!! The log below does not have timestamps," + \
" the Sconscript may not have finished.\n"
sconstruct.write(warning_string)
sconstruct.write(f + '\n')
sconstruct.write(sconscript.read())
return None
def build_matlab(target, source, env):
'''Build targets with a MATLAB command
This function executes a MATLAB function to build objects
specified by target using the objects specified by source.
It requires MATLAB to be callable from the command line
via `matlab`.
Accessing command line arguments from within matlab is
possible via the `command_line_arg = getenv('CL_ARG')`.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the MATLAB .M script that the builder is intended to execute.
env: SCons construction environment, see SCons user guide 7.2
'''
if misc.is_unix():
options = '-nosplash -nodesktop'
elif sys.platform == 'win32':
options = '-nosplash -minimize -wait'
else:
raise PrerequisiteError("Unsupported OS")
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, '.m')
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# Set up command line arguments
cl_arg = misc.command_line_args(env)
os.environ['CL_ARG'] = cl_arg
# Run MATLAB on source file
shutil.copy(source_file, 'source.m')
try:
command = 'matlab %s -r source > %s' % (options, log_file)
subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError:
message = misc.command_error_msg("Matlab", command)
raise ExecCallError(message)
os.remove('source.m')
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_tables(target, source, env):
'''Build a SCons target by filling a table
This function uses the tablefill function from gslab_fill to produced a
filled table from (i) an empty table in a LyX/Tex file and (ii) text files
containing data to be used in filling the table.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the LyX/Tex file specifying the table format. The subsequent
sources should be the text files containing the data with which the
tables are to be filled.
env: SCons construction environment, see SCons user guide 7.2
'''
# Prelims
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
start_time = misc.current_time()
# Set up source file (table format)
source_file = str(source[0])
misc.check_code_extension(source_file, ['.lyx', '.tex'])
# Set up input string (list of data tables)
input_string = ' '.join([str(i) for i in source[1:]])
# Set up target file (filled table)
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
misc.check_code_extension(target_file, ['.lyx', '.tex'])
try:
log_ext = '_%s' % env['log_ext']
except KeyError:
log_ext = ''
log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext))
# Command call
output = tablefill(input=input_string,
template=source_file,
output=target_file)
with open(log_file, 'wb') as f:
f.write(output)
f.write("\n\n")
# Close log
if "traceback" in str.lower(output): # if tablefill.py returns an error
command = """tablefill(input = %s,
template = %s,
output = %s)""" % (input_string, source_file,
target_file)
message = misc.command_error_msg("tablefill.py", command)
raise ExecCallError(message)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
def build_stata(target, source, env):
'''Build targets with a Stata command
This function executes a Stata script to build objects specified
by target using the objects specified by source.
Parameters
----------
target: string or list
The target(s) of the SCons command.
source: string or list
The source(s) of the SCons command. The first source specified
should be the Stata .do script that the builder is intended to
execute.
Note: the user can specify a flavour by typing `scons sf=StataMP`
(By default, SCons will try to find each flavour).
'''
cl_arg = misc.command_line_args(env)
source = misc.make_list_if_string(source)
target = misc.make_list_if_string(target)
source_file = str(source[0])
target_file = str(target[0])
target_dir = misc.get_directory(target_file)
start_time = misc.current_time()
misc.check_code_extension(source_file, 'stata')
log_file = target_dir + '/sconscript.log'
loc_log = os.path.basename(source_file).replace('.do', '.log')
user_flavor = env['user_flavor']
if user_flavor is not None:
if misc.is_unix():
command = misc.stata_command_unix(user_flavor, cl_arg)
elif sys.platform == 'win32':
command = misc.stata_command_win(user_flavor, cl_arg)
else:
flavors = ['stata-mp', 'stata-se', 'stata']
if misc.is_unix():
for flavor in flavors:
if misc.is_in_path(flavor):
command = misc.stata_command_unix(flavor, cl_arg)
break
elif sys.platform == 'win32':
try:
key_exist = os.environ['STATAEXE'] is not None
command = misc.stata_command_win("%%STATAEXE%%")
except KeyError:
flavors = [(f.replace('-', '') + '.exe') for f in flavors]
if misc.is_64_windows():
flavors = [f.replace('.exe', '-64.exe') for f in flavors]
for flavor in flavors:
if misc.is_in_path(flavor):
command = misc.stata_command_win(flavor, cl_arg)
break
try:
subprocess.check_output(command % source_file,
stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError:
raise BadExecutableError('Could not find executable.')
shutil.move(loc_log, log_file)
end_time = misc.current_time()
log_timestamp(start_time, end_time, log_file)
return None
