def build_python(target, source, env): '''Build SCons targets using a Python script This function executes a Python script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Python script that the builder is intended to execute. ''' source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'python') log_file = target_dir + '/sconscript.log' cl_arg = misc.command_line_args(env) os.system('python %s %s > %s' % (source_file, cl_arg, log_file)) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_lyx(target, source, env): '''Compile a pdf from a LyX file This function is a SCons builder that compiles a .lyx file as a pdf and places it at the path specified by target. Parameters ---------- target: string or list The target of the SCons command. This should be the path of the pdf that the builder is instructed to compile. source: string or list The source of the SCons command. This should be the .lyx file that the function will compile as a PDF. ''' source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'lyx') newpdf = source_file.replace('.lyx', '.pdf') log_file = target_dir + '/sconscript.log' os.system('lyx -e pdf2 %s > %s' % (source_file, log_file)) shutil.move(newpdf, target_file) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_tables(target, source, env): '''Build a SCons target by filling a table This function uses the tablefill function from gslab_fill to produced a filled table from (i) an empty table in a LyX file and (ii) text files containing data to be used in filling the table. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the LyX file specifying the table format. The subsequent sources should be the text files containing the data with which the tables are to be filled. ''' source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) misc.check_code_extension(str(target[0]), 'lyx') tablefill(input = ' '.join([str(a) for a in source[1:len(source)]]), template = str(source[0]), output = str(target[0])) return None
def build_lyx(target, source, env): '''Compile a pdf from a LyX file This function is a SCons builder that compiles a .lyx file as a pdf and places it at the path specified by target. Parameters ---------- target: string or list The target of the SCons command. This should be the path of the pdf that the builder is instructed to compile. source: string or list The source of the SCons command. This should be the .lyx file that the function will compile as a PDF. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) misc.check_code_extension(source_file, '.lyx') # Set up target file and log file newpdf = source_file[:-4] + '.pdf' target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'lyx') newpdf = source_file.replace('.lyx', '.pdf') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # System call try: command = 'lyx -e pdf2 %s > %s' % (source_file, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) # Move rendered pdf to the target shutil.move(newpdf, target_file) except subprocess.CalledProcessError: message = misc.command_error_msg("lyx", command) raise ExecCallError(message) # Close log end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def test_make_list_if_string(self): self.assertEqual(misc.make_list_if_string(['test', 'test']), ['test', 'test']) self.assertEqual(misc.make_list_if_string('test'), ['test']) self.assertEqual(misc.make_list_if_string(['test']), ['test']) # We expect the function to raise Type Errors when it receives # inputs that are not strings or lists with self.assertRaises(TypeError): self.assertEqual(misc.make_list_if_string(1), 1) with self.assertRaises(TypeError): self.assertEqual(misc.make_list_if_string(None), None)
def build_stata(target, source, env): '''Build targets with a Stata command This function executes a Stata script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Stata .do script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) start_time = misc.current_time() # Set up source file and the original location of the log source_file = str(source[0]) misc.check_code_extension(source_file, '.do') loc_log = os.path.basename(source_file).replace('.do', '.log') # Set up log file destination target_file = str(target[0]) target_dir = misc.get_directory(target_file) try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # Set up command line arguments cl_arg = misc.command_line_args(env) executable = misc.get_stata_executable(env) command_skeleton = misc.get_stata_command(executable) try: command = command_skeleton % (source_file, cl_arg) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: message = misc.command_error_msg("Stata", command) raise ExecCallError(message) shutil.move(loc_log, log_file) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_r(target, source, env): '''Build SCons targets using an R script This function executes an R script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the R script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'r') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) cl_arg = misc.command_line_args(env) if cl_arg != '': if misc.is_unix(): # R has platform-specific cl_arg syntax cl_arg = "'--args %s'" % cl_arg else: cl_arg = "\"--args %s\"" % cl_arg # System call try: command = 'R CMD BATCH --no-save %s %s %s' % (cl_arg, source_file, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: message = misc.command_error_msg("R", command) raise ExecCallError(message) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def collect_builder_logs(parent_dir, excluded_dirs = []): ''' Recursively return dictionary of files named sconscript*.log in parent_dir and nested directories. Also return timestamp from those sconscript.log (snippet from SO 3964681) excluded_dirs (str or list of str): list of directories to be excluded from the search ''' builder_log_collect = {} # Store paths to logs in a list, found from platform-specific command line tool rel_parent_dir = os.path.relpath(parent_dir) log_name = '*sconscript*.log' excluded_dirs = misc.make_list_if_string(excluded_dirs) log_paths = misc.finder(rel_parent_dir, log_name, excluded_dirs) # Read the file at each path to a log and store output complete-time in a dict at filename for log_path in log_paths: with open(log_path, 'rU') as f: try: s = f.readlines()[1] # line 0 = log start time, line 1 = log end time except IndexError: s = '' s = s[s.find('{') + 1: s.find('}')] # find {} time identifier try: builder_log_end_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S") except ValueError: # if the code breaks, there's no time identifier beginning_of_time = datetime.min builder_log_end_time = beginning_of_time builder_log_collect[log_path] = builder_log_end_time return builder_log_collect
def build_python(target, source, env): '''Build SCons targets using a Python script This function executes a Python script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Python script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, '.py') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) cl_arg = misc.command_line_args(env) # System call try: command = 'python %s %s > %s' % (source_file, cl_arg, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as ex: message = misc.command_error_msg("Python", command) raise ExecCallError('%s\n%s' % (message, ex.output)) # Close log end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def __init__(self, target, source, env, name='GSLab Builder', valid_extensions=[], exec_opts=''): ''' Fill builder with information about build step. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 name: string Name of builder-type. Use to refer to builder in error messages and env. valid_extensions: iterable Valid (case-insensitive) extensions for first element of source list. default_exec: string Executable used to execute first element of source list. Override by passing value through env. exec_opts: string Options used to execute first element of source list. ''' # Store keyword args self.name = name self.valid_extensions = valid_extensions self.exec_opts = exec_opts # Build system call and store components self.source_file = str(misc.make_list_if_string(source)[0]) self.target = [str(t) for t in misc.make_list_if_string(target)] self.target_dir = misc.get_directory(self.target[0]) if 'executable_names' not in env: env['executable_names'] = {} self.executable = misc.get_executable(name, env['executable_names']) self.env = env self.add_command_line_arg() self.add_log_file() self.add_call_args() self.system_call = '%s %s %s' % (self.executable, self.exec_opts, self.call_args) return None
def build_anything(target, source, action, env, warning=True, **kw): ''' Anything builder-generator. The generator will create a custom builder that runs `action` and add it as a SCons node, similar to the native env.Command. Using gslab_scons.build_anything will utilize our logging mechanism and error catching similar to our other builders. ` Parameters target: string or list The target(s) of the SCons command. The log ends up in the directory of the first specified target. source: string or list The source(s) of the SCons command. action: string The code to be run by the generated builder. env: SCons construction environment, see SCons user guide 7.2. You *** MUST *** manually pass `env = env` when calling this in SConscript, since this is not a Scons.env method like env.Command. Special parameters that can be added when using the builder log_ext: string Instead of logging to `sconscript.log` in the target dir, the builder will log to `sconscript_<log_ext>.log`. origin_log_file: string Sometimes, your command may produce a log file in an undesirable location. Specifying the that location in this argument leads the builder to append the content of origin_log_file to log_file and delete origin_log_file. The builder will crash if this file doesn't exist at the end of the command. warning: Boolean Turns off warnings if warning = False. ''' import SCons.Builder builder_attributes = {'name': 'Anything Builder'} target = [t for t in misc.make_list_if_string(target) if t] source = [s for s in misc.make_list_if_string(source) if s] local_env = env.Clone() for k, v in kw.items(): local_env[k] = v builder = AnythingBuilder(target, source, action, local_env, warning, **builder_attributes) bkw = { 'action': builder.build_anything, 'target_factory': local_env.fs.Entry, 'source_factory': local_env.fs.Entry, } bld = SCons.Builder.Builder(**bkw) return bld(local_env, target, source)
def __init__(self, target, source, action, env, warning=True, name=''): ''' ''' target = [self.to_str(t) for t in misc.make_list_if_string(target)] source = [self.to_str(s) for s in misc.make_list_if_string(source)] self.action = action super(AnythingBuilder, self).__init__(target, source, env, name=name) try: origin_log_file = env['origin_log_file'] except KeyError: origin_log_file = None self.origin_log_file = origin_log_file if '>' in action and warning == True: warning_message = '\nThere is a redirection operator > in ' \ 'your prescribed action key.\n' \ 'The Anything Builder\'s logging mechanism '\ 'may not work as intended.' warnings.warn(warning_message) return None
def add_source_file(self, source): ''' Add source file to execute as the first element of source. If source is an empty list, then the source file is ''. ''' if bool(source): sources = misc.make_list_if_string(source) source_file = str(sources[0]) else: source_file = '' self.source_file = os.path.normpath("%s" % source_file) return None
def test_make_list_if_string(self): self.assertEqual(misc.make_list_if_string(['test', 'test']), ['test', 'test']) self.assertEqual(misc.make_list_if_string('test'), ['test']) self.assertEqual(misc.make_list_if_string(['test']), ['test']) # We expect objects that are neither strings nor lists to be # returned without being manipulated. self.assertEqual(misc.make_list_if_string(1), 1) self.assertEqual(misc.make_list_if_string(TypeError), TypeError) self.assertEqual(misc.make_list_if_string(False), False) self.assertEqual(misc.make_list_if_string(None), None)
def check_code_extension(self): ''' Raise an exception if the extension in executing script does not mach extension in valid_extensions attribute. ''' extensions = misc.make_list_if_string(self.valid_extensions) if extensions == []: return None matches = [True for extension in extensions if self.source_file.lower().endswith("%s" % extension)] if not matches: message = 'First argument, %s, must be a file of type %s.' % (self.source_file, extensions) raise BadExtensionError(message) return None
def add_command_line_arg(self): ''' Store arguments to pass to the executing script on the command line. Return the content of env['CL_ARG'] as a string with spaces separating entries. If env['CL_ARG'] doesn't exist, return an empty string. ''' try: cl_arg = self.env['CL_ARG'] except KeyError: cl_arg = '' try: cl_arg = misc.make_list_if_string(cl_arg) cl_arg = ' '.join([str(s) for s in cl_arg]) except TypeError: cl_arg = str(cl_arg) self.cl_arg = "%s" % cl_arg return None
def build_matlab(target, source, env): '''Build targets with a MATLAB command This function executes a MATLAB function to build objects specified by target using the objects specified by source. It requires MATLAB to be callable from the command line via `matlab`. Accessing command line arguments from within matlab is possible via the `command_line_arg = getenv('CL_ARG')`. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the MATLAB .M script that the builder is intended to execute. env: SCons construction environment, see SCons user guide 7.2 ''' if misc.is_unix(): options = '-nosplash -nodesktop' elif sys.platform == 'win32': options = '-nosplash -minimize -wait' else: raise PrerequisiteError("Unsupported OS") source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, '.m') try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # Set up command line arguments cl_arg = misc.command_line_args(env) os.environ['CL_ARG'] = cl_arg # Run MATLAB on source file shutil.copy(source_file, 'source.m') try: command = 'matlab %s -r source > %s' % (options, log_file) subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: message = misc.command_error_msg("Matlab", command) raise ExecCallError(message) os.remove('source.m') end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_stata(target, source, env): '''Build targets with a Stata command This function executes a Stata script to build objects specified by target using the objects specified by source. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the Stata .do script that the builder is intended to execute. Note: the user can specify a flavour by typing `scons sf=StataMP` (By default, SCons will try to find each flavour). ''' cl_arg = misc.command_line_args(env) source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) source_file = str(source[0]) target_file = str(target[0]) target_dir = misc.get_directory(target_file) start_time = misc.current_time() misc.check_code_extension(source_file, 'stata') log_file = target_dir + '/sconscript.log' loc_log = os.path.basename(source_file).replace('.do', '.log') user_flavor = env['user_flavor'] if user_flavor is not None: if misc.is_unix(): command = misc.stata_command_unix(user_flavor, cl_arg) elif sys.platform == 'win32': command = misc.stata_command_win(user_flavor, cl_arg) else: flavors = ['stata-mp', 'stata-se', 'stata'] if misc.is_unix(): for flavor in flavors: if misc.is_in_path(flavor): command = misc.stata_command_unix(flavor, cl_arg) break elif sys.platform == 'win32': try: key_exist = os.environ['STATAEXE'] is not None command = misc.stata_command_win("%%STATAEXE%%") except KeyError: flavors = [(f.replace('-', '') + '.exe') for f in flavors] if misc.is_64_windows(): flavors = [f.replace('.exe', '-64.exe') for f in flavors] for flavor in flavors: if misc.is_in_path(flavor): command = misc.stata_command_win(flavor, cl_arg) break try: subprocess.check_output(command % source_file, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError: raise BadExecutableError('Could not find executable.') shutil.move(loc_log, log_file) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None
def build_tables(target, source, env): '''Build a SCons target by filling a table This function uses the tablefill function from gslab_fill to produced a filled table from (i) an empty table in a LyX/Tex file and (ii) text files containing data to be used in filling the table. Parameters ---------- target: string or list The target(s) of the SCons command. source: string or list The source(s) of the SCons command. The first source specified should be the LyX/Tex file specifying the table format. The subsequent sources should be the text files containing the data with which the tables are to be filled. env: SCons construction environment, see SCons user guide 7.2 ''' # Prelims source = misc.make_list_if_string(source) target = misc.make_list_if_string(target) start_time = misc.current_time() # Set up source file (table format) source_file = str(source[0]) misc.check_code_extension(source_file, ['.lyx', '.tex']) # Set up input string (list of data tables) input_string = ' '.join([str(i) for i in source[1:]]) # Set up target file (filled table) target_file = str(target[0]) target_dir = misc.get_directory(target_file) misc.check_code_extension(target_file, ['.lyx', '.tex']) try: log_ext = '_%s' % env['log_ext'] except KeyError: log_ext = '' log_file = os.path.join(target_dir, ('sconscript%s.log' % log_ext)) # Command call output = tablefill(input=input_string, template=source_file, output=target_file) with open(log_file, 'wb') as f: f.write(output) f.write("\n\n") # Close log if "traceback" in str.lower(output): # if tablefill.py returns an error command = """tablefill(input = %s, template = %s, output = %s)""" % (input_string, source_file, target_file) message = misc.command_error_msg("tablefill.py", command) raise ExecCallError(message) end_time = misc.current_time() log_timestamp(start_time, end_time, log_file) return None