def build_lyx(target, source, env): '''Compile a pdf from a LyX file This function is a SCons builder that compiles a .lyx file as a pdf and places it at the path specified by target. Parameters ---------- target: string or list The target of the SCons command. This should be the path of the pdf that the builder is instructed to compile. source: string or list The source of the SCons command. This should be the .lyx file that the function will compile as a PDF. ''' source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'lyx') newpdf = source_file.replace('.lyx', '.pdf') log_file = target_dir + '/sconscript.log' os.system('lyx -e pdf2 %s > %s' % (source_file, log_file)) shutil.move(newpdf, target_file) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_python(target, source, env): '''Build SCons targets using a Python script This function executes a Python script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Python script that the builder is intended to execute. ''' source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'python') log_file = target_dir + '/sconscript.log' cl_arg = misc.command_line_args(env) os.system('python %s %s > %s' % (source_file, cl_arg, log_file)) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_lyx(target, source, env): '''Compile a pdf from a LyX file This function is a SCons builder that compiles a .lyx file as a pdf and places it at the path specified by target. Parameters ---------- target: string or list The target of the SCons command. This should be the path of the pdf that the builder is instructed to compile. source: string or list The source of the SCons command. This should be the .lyx file that the function will compile as a PDF. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) misc.check_code_extension(source_file, '.lyx') # Set up target file and log file newpdf = source_file[:-4] + '.pdf' target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'lyx') newpdf = source_file.replace('.lyx', '.pdf') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # System call try: command = 'lyx -e pdf2 %s > %s' % (source_file, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) # Move rendered pdf to the target shutil.move(newpdf, target_file) except subprocess.CalledProcessError: message = misc.command_error_msg("lyx", command) raise ExecCallError(message) # Close log end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_stata(target, source, env): '''Build targets with a Stata command This function executes a Stata script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Stata .do script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) start_time = misc.current_time() # Set up source file and the original location of the log source_file = str(source[0]) misc.check_code_extension(source_file, '.do') loc_log = os.path.basename(source_file).replace('.do', '.log') # Set up log file destination target_file = str(target[0]) target_dir = misc.get_directory(target_file) try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # Set up command line arguments cl_arg = misc.command_line_args(env) executable = misc.get_stata_executable(env) command_skeleton = misc.get_stata_command(executable) try: command = command_skeleton % (source_file, cl_arg) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: message = misc.command_error_msg("Stata", command) raise ExecCallError(message) shutil.move(loc_log, log_file) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_r(target, source, env): '''Build SCons targets using an R script This function executes an R script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the R script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'r') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) cl_arg = misc.command_line_args(env) if cl_arg != '': if misc.is_unix(): # R has platform-specific cl_arg syntax cl_arg = "'--args %s'" % cl_arg else: cl_arg = "\"--args %s\"" % cl_arg # System call try: command = 'R CMD BATCH --no-save %s %s %s' % (cl_arg, source_file, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: message = misc.command_error_msg("R", command) raise ExecCallError(message) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def execute_system_call(self): ''' Execute the system call attribute. Log the execution. Check that expected targets exist after execution. ''' self.check_code_extension() start_time = misc.current_time() self.do_call() self.check_targets() end_time = misc.current_time() self.timestamp_log(start_time, end_time) return None
def build_python(target, source, env): '''Build SCons targets using a Python script This function executes a Python script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Python script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, '.py') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) cl_arg = misc.command_line_args(env) # System call try: command = 'python %s %s > %s' % (source_file, cl_arg, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as ex: message = misc.command_error_msg("Python", command) raise ExecCallError('%s\n%s' % (message, ex.output)) # Close log end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def test_current_time(self): ''' Test that current_time() prints the correct current time in the expected format. Here, the "correct current time" is a fixed value set within the MockDateTime class. ''' the_time = misc.current_time() self.assertEqual('2017-01-20 15:02:18', the_time)
def end_log(cl_args_list=sys.argv, log='sconstruct.log', excluded_dirs=[], release_dir='./release/'): '''Complete the log of a build process.''' if misc.is_scons_dry_run(cl_args_list=cl_args_list): return None end_message = "*** Build completed: {%s} ***\n \n \n" % misc.current_time() with open(log, "a") as f: f.write(end_message) # scan sconstruct.log for start time with open(log, "rU") as f: s = f.readline() s = s[s.find('{') + 1:s.find('}')] start_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S") # gather all sconscript logs parent_dir = os.getcwd() builder_logs = collect_builder_logs(parent_dir, excluded_dirs=excluded_dirs) # keep only builder logs from this run OR is broken (value == beginning_of_time) beginning_of_time = datetime.min # to catch broken logs (see collect_builder_logs) this_run_dict = { key: value for key, value in builder_logs.items() if (value > start_time) or value == beginning_of_time } this_run_list = sorted(this_run_dict, key=this_run_dict.get, reverse=True) with open(log, "a") as sconstruct: for f in this_run_list: with open(f, 'rU') as sconscript: if this_run_dict[f] == beginning_of_time: warning_string = "*** Warning!!! The log below does not have timestamps," + \ " the Sconscript may not have finished.\n" sconstruct.write(warning_string) sconstruct.write(f + '\n') sconstruct.write(sconscript.read()) # move top level logs to /release/ directory. if not os.path.exists(release_dir): os.makedirs(release_dir) for file in glob.glob("*.log"): shutil.move('./' + file, release_dir + file) return None
def start_log(mode, vers, log='sconstruct.log'): '''Begins logging a build process''' if not (mode in ['develop', 'cache']): raise Exception("Error: %s is not a defined mode" % mode) start_message = "*** New build: {%s} ***\n" % misc.current_time() with open(log, "w") as f: f.write(start_message) if misc.is_unix(): sys.stdout = os.popen('tee -a %s' % log, 'wb') elif sys.platform == 'win32': sys.stdout = open(log, 'ab') sys.stderr = sys.stdout return None
def end_log(log='sconstruct.log'): '''Complete the log of a build process.''' end_message = "*** Build completed: {%s} ***\n \n \n" % misc.current_time() with open(log, "a") as f: f.write(end_message) # scan sconstruct.log for start time with open('sconstruct.log', "rU") as f: s = f.readline() s = s[s.find('{') + 1:s.find('}')] start_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S") # gather all sconscript logs parent_dir = os.getcwd() builder_logs = collect_builder_logs(parent_dir) # keep only builder logs from this run OR is broken (value == beginning_of_time) beginning_of_time = datetime.min # to catch broken logs (see collect_builder_logs) this_run_dict = { key: value for key, value in builder_logs.items() if (value > start_time) or value == beginning_of_time } this_run_list = sorted(this_run_dict, key=this_run_dict.get, reverse=True) with open('sconstruct.log', "a") as sconstruct: for f in this_run_list: with open(f, 'rU') as sconscript: if this_run_dict[f] == beginning_of_time: warning_string = "*** Warning!!! The log below does not have timestamps," + \ " the Sconscript may not have finished.\n" sconstruct.write(warning_string) sconstruct.write(f + '\n') sconstruct.write(sconscript.read()) return None
def build_matlab(target, source, env): '''Build targets with a MATLAB command This function executes a MATLAB function to build objects specified by target using the objects specified by source. It requires MATLAB to be callable from the command line via `matlab`. Accessing command line arguments from within matlab is possible via the `command_line_arg = getenv('CL_ARG')`. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the MATLAB .M script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' if misc.is_unix(): options = '-nosplash -nodesktop' elif sys.platform == 'win32': options = '-nosplash -minimize -wait' else: raise PrerequisiteError("Unsupported OS") source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, '.m') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # Set up command line arguments cl_arg = misc.command_line_args(env) os.environ['CL_ARG'] = cl_arg # Run MATLAB on source file shutil.copy(source_file, 'source.m') try: command = 'matlab %s -r source > %s' % (options, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: message = misc.command_error_msg("Matlab", command) raise ExecCallError(message) os.remove('source.m') end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_tables(target, source, env): '''Build a SCons target by filling a table This function uses the tablefill function from gslab_fill to produced a filled table from (i) an empty table in a LyX/Tex file and (ii) text files containing data to be used in filling the table. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the LyX/Tex file specifying the table format. The subsequent sources should be the text files containing the data with which the tables are to be filled. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) start_time = misc.current_time() # Set up source file (table format) source_file = str(source[0]) misc.check_code_extension(source_file, ['.lyx', '.tex']) # Set up input string (list of data tables) input_string = ' '.join([str(i) for i in source[1:]]) # Set up target file (filled table) target_file = str(target[0]) target_dir = misc.get_directory(target_file) misc.check_code_extension(target_file, ['.lyx', '.tex']) try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # Command call output = tablefill(input=input_string, template=source_file, output=target_file) with open(log_file, 'wb') as f: f.write(output) f.write("\n\n") # Close log if "traceback" in str.lower(output): # if tablefill.py returns an error command = """tablefill(input = %s, template = %s, output = %s)""" % (input_string, source_file, target_file) message = misc.command_error_msg("tablefill.py", command) raise ExecCallError(message) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_stata(target, source, env): '''Build targets with a Stata command This function executes a Stata script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Stata .do script that the builder is intended to execute. Note: the user can specify a flavour by typing `scons sf=StataMP` (By default, SCons will try to find each flavour). ''' cl_arg = misc.command_line_args(env) source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'stata') log_file = target_dir + '/sconscript.log' loc_log = os.path.basename(source_file).replace('.do', '.log') user_flavor = env['user_flavor'] if user_flavor is not None: if misc.is_unix(): command = misc.stata_command_unix(user_flavor, cl_arg) elif sys.platform == 'win32': command = misc.stata_command_win(user_flavor, cl_arg) else: flavors = ['stata-mp', 'stata-se', 'stata'] if misc.is_unix(): for flavor in flavors: if misc.is_in_path(flavor): command = misc.stata_command_unix(flavor, cl_arg) break elif sys.platform == 'win32': try: key_exist = os.environ['STATAEXE'] is not None command = misc.stata_command_win("%%STATAEXE%%") except KeyError: flavors = [(f.replace('-', '') + '.exe') for f in flavors] if misc.is_64_windows(): flavors = [f.replace('.exe', '-64.exe') for f in flavors] for flavor in flavors: if misc.is_in_path(flavor): command = misc.stata_command_win(flavor, cl_arg) break try: subprocess.check_output(command % source_file, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: raise BadExecutableError('Could not find executable.') shutil.move(loc_log, log_file) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None