def __call__(self, target, source, env): """Both target and source should be a single file""" if len(target) != 1: raise StopError("unexpected number of targets for unitTest: " + str(target)) if len(source) != 1: raise StopError("unexpected number of sources for unitTest: " + str(source)) out = str(target[0]) exe = str(source[0]) # additional envvars to pass to command cmdenv = [] # On Mac OS X El Capitan we will need to side load the library load path # and fix up the binary rpath if platform_default() == 'darwin': # El Capitan is Darwin 15.x. We could simply always set this # variable on OS X rather than restricting to version as # in the future it will be more likely than not that we # are running a SIP platform. release_str = platform.release() release_major = int(release_str.split('.')[0]) if release_major >= 15: if "LSST_LIBRARY_PATH" in os.environ: os.environ["DYLD_LIBRARY_PATH"] = os.environ[ "LSST_LIBRARY_PATH"] # We need to tell the tests where the uninstalled Qserv libraries # are located because rpaths aren't (yet?) handled properly # when the libraries and binaries are built on OS X. cmdenv.append("{}='{}:{}'".format("DYLD_LIBRARY_PATH", env["build_dir"], os.environ["DYLD_LIBRARY_PATH"])) # pass logging config file if defined log_config = env.get("UNIT_TEST_LOG_CONFIG") if log_config: cmdenv.append("LSST_LOG_CONFIG={}".format(log_config)) try: cmd = "{} {} > {} 2>&1".format(" ".join(cmdenv), exe, out) ret = os.system(cmd) if ret != 0: shutil.move(out, out + '.failed') msg = '*** Unit test failed, check log file ' + out + '.failed ***' stars = '*' * len(msg) print(stars, msg, stars, sep="\n") # save failed target in UNIT_TESTS_FAILED list in env, # to be analyzed by UnitTestCheck env.Append(UNIT_TESTS_FAILED=source) except: # exception means we could not even run it print('Failure running unit test ' + out) env.Append(UNIT_TESTS_FAILED=source)
def generate(env): env.AppendUnique(CXXPATH=["/usr/include/catch"]) if not exists(env): raise StopError("Catch header must be found in $CPPPATH/$CXXPATH/$CPATH") catch_test = Builder(generator=generate_catch_test_run) env.AppendUnique(BUILDERS=dict(CatchTest=catch_test))
def _get_unit_name(env, unit=None): """ Retrieve the full name (relative path from the project root) of a unit. A 'unit' is an buildable component that can be referenced and required as a dependency of another buildable component. Although Sconscripts can reference other units by a shortened path, internally they are referenced using the relative path from the root to avoid conflicts. If the unit keyword parameter is not given then the name of the unit calling this function is returned. """ # Given unit name can be given in shortened form. Search from the # current directory to find the intended unit. start_dir = env.Dir('.').srcnode().abspath # Determine the name of the calling unit when none is given. if unit is None: unit = os.path.basename(start_dir) # Find the unit and return the name (relative path from the root). root_dir = _get_project_dir(env) unit = unit.replace('/', os.path.sep) search_dir = start_dir while True: unit_dir = os.path.join(search_dir, unit) if os.path.exists(os.path.join(unit_dir, 'SConscript')): return os.path.relpath(unit_dir, root_dir) if search_dir == root_dir: raise StopError('Could not find unit %s' % unit) search_dir = os.path.normpath(os.path.join(search_dir, os.path.pardir))
def get_base_env(*args, **kwargs): """Initialize and return a base construction environment. All args received are passed transparently to SCons Environment init. """ # Initialize new construction environment env = Environment(*args, **kwargs) # pylint: disable=undefined-variable # If a flavor is activated in the external environment - use it if 'BUILD_FLAVOR' in os.environ: active_flavor = os.environ['BUILD_FLAVOR'] if not active_flavor in flavors(): raise StopError('%s (from env) is not a known flavor.' % (active_flavor)) sprint('Using active flavor "%s" from your environment', active_flavor) env.flavors = [active_flavor] else: # If specific flavor target specified, skip processing other flavors # Otherwise, include all known flavors env.flavors = ( set(flavors()).intersection(COMMAND_LINE_TARGETS) # pylint: disable=undefined-variable or flavors()) # Perform base construction environment customizations from site_config if '_common' in ENV_OVERRIDES: env.Replace(**ENV_OVERRIDES['_common']) if '_common' in ENV_EXTENSIONS: env.Append(**ENV_EXTENSIONS['_common']) return env
def parse(self, test_f): wholeline = '' lnum = 0 error = False for line in test_f: line = line.strip() lnum += 1 # Ignore whitespace lines and comments if not line or line.startswith('#'): continue if line.endswith('\\'): line = line[:-1].strip() wholeline += line + ' ' continue wholeline += line try: if wholeline.startswith('^'): self.parse_group(wholeline[1:]) else: self.parse_test(wholeline) wholeline = '' except (TestException, ValueError) as e: error = True print >> sys.stderr, 'Error on line %d:' % lnum, str(e) print >> sys.stderr, '\t\t ' + wholeline if error: raise StopError('Failed to parse test suite config!')
def build(self): """Build flavor using three-pass strategy.""" # First pass - compile protobuffers for module in modules(): # Verify the SConscript file exists sconscript_path = os.path.join(module, 'SConscript') if not os.path.isfile(sconscript_path): raise StopError('Missing SConscript file for module %s.' % (module)) sprint('|- First pass: Reading module %s ...', module) shortcuts = dict( Lib=nop, StaticLib=nop, SharedLib=nop, Proto=self._proto_wrapper(), Prog=nop, ) self._env.SConscript(sconscript_path, variant_dir=os.path.join( '$BUILDROOT', module), exports=shortcuts) #Second pass over all modules - process and collect library targets for module in modules(): shortcuts = dict( Lib=self._lib_wrapper(self._env.Library, module), StaticLib=self._lib_wrapper(self._env.StaticLibrary, module), SharedLib=self._lib_wrapper(self._env.SharedLibrary, module), Proto=nop, Prog=nop, ) self._env.SConscript(os.path.join(module, 'SConscript'), variant_dir=os.path.join( '$BUILDROOT', module), exports=shortcuts) # Third pass over all modules - process program targets shortcuts = dict() for nop_shortcut in ('Lib', 'StaticLib', 'SharedLib', 'Proto'): shortcuts[nop_shortcut] = nop for module in modules(): sprint('|- Second pass: Reading module %s ...', module) shortcuts['Prog'] = self._prog_wrapper(module) self._env.SConscript(os.path.join(module, 'SConscript'), variant_dir=os.path.join( '$BUILDROOT', module), exports=shortcuts) # Add install targets for programs from all modules for module, prog_nodes in self._progs.iteritems(): for prog in prog_nodes: assert isinstance(prog, Node.FS.File) # If module is hierarchical, replace pathseps with periods bin_name = path_to_key('%s' % (prog.name)) self._env.InstallAs(os.path.join('$BINDIR', bin_name), prog) # Support using the flavor name as target name for its related targets self._env.Alias(self._flavor, '$BUILDROOT')
def _detect(env): """ Helper function to detect the astyle excecutable.""" astyle = env.get('ASTYLE', env.WhereIs('astyle')) if astyle: return astyle else: raise StopError( AstyleCompilerNotFound, "Could not detect ASTYLE") # surely we could detect the platform
def _get_project_dir(env): """ Retrieve the absolute path of the project (i.e. SConstruct) directory. """ project_dir = env.Dir('.').srcnode().abspath while not os.path.exists(os.path.join(project_dir, 'SConstruct')): parent_dir = os.path.normpath(os.path.join(project_dir, os.path.pardir)) if parent_dir == project_dir: raise StopError('Could not find project root (i.e. SConstruct).') project_dir = parent_dir return project_dir
def build_prog(prog_name, sources=None, with_libs=None, **kwargs): """Customized program builder. @param prog_name Program name @param sources Source file (or list of source files) @param with_libs Library name (or list of library names) to link with. kwargs params: @param install Binary flag to override default value from closure (`default_install`). @param protos Names of proto (or protos) to add to target. """ # Extend sources list with protos from generated code manager sources = self._extend_proto_sources(sources, kwargs) install_flag = kwargs.pop('install', default_install) # Process library dependencies - add libs specified in `with_libs` for lib_name in listify(with_libs): lib_keys = listify(self._get_matching_lib_keys(lib_name)) if len(lib_keys) == 1: # Matched internal library lib_key = lib_keys[0] # Extend prog sources with library nodes sources.extend(self._libs[lib_key]) elif len(lib_keys) > 1: # Matched multiple internal libraries - probably bad! raise StopError( 'Library identifier "%s" matched %d ' 'libraries (%s). Please use a fully ' 'qualified identifier instead!' % (lib_name, len(lib_keys), ', '.join(lib_keys))) else: # empty lib_keys raise StopError('Library identifier "%s" didn\'t match ' 'any library. Is it a typo?' % (lib_name)) # Build the program and add to prog nodes dict if installable prog_nodes = self._env.Program(prog_name, sources, **kwargs) if install_flag: # storing each installable node in a dictionary instead of # defining InstallAs target on the spot, because there's # an "active" variant dir directive messing with paths. self._progs[module].extend(prog_nodes)
def _astyle_check_action(target, source, env): '''This prepares the environment for _astyle_check_diff to run''' # Get the report file. report_file = target[0].abspath # Get the output directory. output_directory = os.path.split(report_file)[0] # Check if the directory exists. if not source: raise StopError('[ERROR] No source files to check') if not os.path.exists(output_directory): os.makedirs(output_directory) # Check if some file need astyle. check_astyle_result = _get_astyle_diff(env, source, output_directory) # Check if _get_astyle_diff() fails. if check_astyle_result is None: raise StopError('[ERROR] Failed running Astyle Check.') # Open the report file. try: report = open(report_file, 'w') except IOError: raise StopError('[ERROR] No such file or directory.') else: # If we can open it we truncate it. report.truncate(0) # If some file needs astyle we print info. if check_astyle_result: # Print a warning message. Cprint('[WARNING] The following files need astyle:', 'yellow') # Print what need to be astyled. for f, info in check_astyle_result: # Write into hte report file. report.write(info + '\n\n') # Print on the screen. Cprint('====> %s' % f, 'yellow') Cprint(info, 'yellow') else: Cprint('[OK] No file needs astyle.', 'green') # Close the report file. report.close() return os.EX_OK
def build(self): """Build flavor using two-pass strategy.""" # First pass over all modules - process and collect library targets for module in modules(): # Verify the SConscript file exists sconscript_path = os.path.join(module, 'SConscript') if not os.path.isfile(sconscript_path): raise StopError( 'Missing SConscript file for module {}.'.format(module)) log_info('|- First pass: Reading module "{}" ...'.format(module)) # log_warn('|- variant_dir "{}" '.format(os.path.join( '$BUILDROOT', module))) shortcuts = dict( Lib=self._lib_wrapper(self._env.Library, module), StaticLib=self._lib_wrapper(self._env.StaticLibrary, module), SharedLib=self._lib_wrapper(self._env.SharedLibrary, module), Prog=dummy_op, env=self._env, ) # Access a protected member of another namespace, # using an undocumented feature of SCons SCons.Script._SConscript.GlobalDict.update(shortcuts) # pylint: disable=protected-access self._env.SConscript(sconscript_path, variant_dir=os.path.join( '$BUILDROOT', module), duplicate=0, exports=shortcuts) # Second pass over all modules - process program targets shortcuts = dict(env=self._env) for nop_shortcut in ('Lib', 'StaticLib', 'SharedLib'): shortcuts[nop_shortcut] = dummy_op for module in modules(): log_info('|- Second pass: Reading module "{}" ...'.format(module)) shortcuts['Prog'] = self._prog_wrapper(module) # Access a protected member of another namespace, # using an undocumented feature of SCons SCons.Script._SConscript.GlobalDict.update(shortcuts) # pylint: disable=protected-access self._env.SConscript(os.path.join(module, 'SConscript'), variant_dir=os.path.join( '$BUILDROOT', module), duplicate=0, exports=shortcuts) if _AUTO_INSTALL_EXE: self.install_progs() if _DEFAULT_FINISHING: self.finishing_progs() # Support using the flavor name as target name for its related targets self._env.Alias(self._flavor, '$BUILDROOT')
def compile_proto(proto_sources, **kwargs): """Customized Protoc builder. Uses Protoc builder to compile proto files specified in `proto_sources`. Optionally pass `cpp=False` to disable C++ code generation. Optionally, pass `python=True` to enable Python code generation. Optionally pass `PROTOPATH=[...]` to override default list of proto search paths (default is [$BUILDROOT]). Optionally pass `PROTOCPPOUT=path` and `PROTOPYOUT=path` to override default output path for C++ / Python outputs (respectively), when output for this language is enabled. Default output paths are $BUILDROOT (so expect output files in the module directory under the flavor build dir). Tip: Don't mess with these... """ if not hasattr(self._env, 'Protoc'): raise StopError('Protoc tool not installed.') # use user-specified value, or set default kwargs.setdefault('PROTOPATH', ['$BUILDROOT']) any_output = False for gen_flag_name, default_gen_flag, path_name, default_path in [ ('cpp', True, 'PROTOCPPOUT', '$BUILDROOT'), ('python', True, 'PROTOPYOUT', '$BUILDROOT'), ]: gen_output_flag = kwargs.pop(gen_flag_name, default_gen_flag) if gen_output_flag: any_output = True # use user-specified value, or set default kwargs.setdefault(path_name, default_path) else: kwargs[path_name] = '' # check that at least one output language is enabled if any_output: targets = self._env.Protoc([], proto_sources, **kwargs) for gen_node in targets: gen_filename = os.path.basename(gen_node.path) if gen_filename.endswith('.pb.cc'): # Save generated .pb.cc sources in proto_cc dictionary # (without the ".pb.cc" suffix) self._proto_cc[gen_filename[:-6]] = gen_node else: sprint('warning: Proto target with no output directives')
def build(self): """Build flavor using two-pass strategy.""" # First pass over all modules - process and collect library targets for module in modules(): # get only the module name (not the path) moduleName = os.path.basename(os.path.normpath(module)) # Verify the SConscript file exists sconscript_path = os.path.join(module, 'SConscript') if not os.path.isfile(sconscript_path): raise StopError('Missing SConscript file for module %s.' % (module)) sprint('|- First pass: Reading module %s ...', module) shortcuts = dict( Lib=self._lib_wrapper(self._env.Library, module), StaticLib=self._lib_wrapper(self._env.StaticLibrary, module), SharedLib=self._lib_wrapper(self._env.SharedLibrary, module), Prog=nop, ) SCons.Script._SConscript.GlobalDict.update(shortcuts) # pylint: disable=protected-access self._env.SConscript(sconscript_path, variant_dir=os.path.join( '$BUILDROOT', moduleName)) # Second pass over all modules - process program targets shortcuts = dict() for nop_shortcut in ('Lib', 'StaticLib', 'SharedLib'): shortcuts[nop_shortcut] = nop for module in modules(): moduleName = os.path.basename(os.path.normpath(module)) sprint('|- Second pass: Reading module %s ...', module) shortcuts['Prog'] = self._prog_wrapper(module) SCons.Script._SConscript.GlobalDict.update(shortcuts) # pylint: disable=protected-access self._env.SConscript(os.path.join(module, 'SConscript'), variant_dir=os.path.join( '$BUILDROOT', moduleName)) # Add install targets for programs from all modules for module, prog_nodes in self._progs.iteritems(): moduleName = os.path.basename(os.path.normpath(module)) for prog in prog_nodes: assert isinstance(prog, Node.FS.File) # If module is hierarchical, replace pathseps with periods bin_name = path_to_key('%s.%s' % (moduleName, prog.name)) self._env.InstallAs(os.path.join('$BINDIR', bin_name), prog)
def jinja_scanner(node, env, path): # Instantiate the file as necessary node.get_text_contents() node_dir = os.path.dirname(str(node)) template_dir, filename = os.path.split(str(node)) template_search_path = ([template_dir] + env.subst(env['JINJA_TEMPLATE_SEARCHPATH'])) template_loader = FileSystemLoaderRecorder(template_search_path) jinja_env = jinja2.Environment(loader=template_loader, extensions=['jinja2.ext.do'], **env['JINJA_ENVIRONMENT_VARS']) try: template = jinja_env.get_template(filename) except TemplateNotFound as e: raise StopError('Missing template: ' + os.path.join(template_dir, str(e))) # We need to render the template to do all the necessary loading. # # It's necessary to respond to missing templates by grabbing # the content as the exception is raised. This makes sure of the # existence of the file upon which the current scanned node depends. # # I suspect that this is pretty inefficient, but it does # work reliably. context = env['JINJA_CONTEXT'] last_missing_file = '' while True: try: template.render(**context) except TemplateNotFound as e: if last_missing_file == str(e): # We've already been round once for this file, # so need to raise raise StopError('Missing template: ' + os.path.join(template_dir, str(e))) last_missing_file = str(e) # Find where the template came from (using the same ordering # as Jinja uses). for searchpath in template_search_path: filename = os.path.join(searchpath, last_missing_file) if os.path.exists(filename): continue else: env.File(filename).get_text_contents() continue break # Get all the files that were loaded. The set includes the current node, # so we remove that. found_nodes_names = list(template_loader.loaded_filenames) try: found_nodes_names.remove(str(node)) except ValueError as e: raise StopError('Missing template node: ' + str(node)) return [env.File(f) for f in found_nodes_names]
tgtroot = 'tsload-doc' Import('env') doc_format = GetOption('doc_format') if doc_format == 'html': doc_suffix = '.html' elif doc_format == 'markdown': doc_suffix = '.md' elif doc_format == 'latex': doc_suffix = '.tex' elif doc_format == 'creole': doc_suffix = '.creole' else: raise StopError("Invalid documentation format '%s'" % doc_format) env['TSDOC_DOCSPACES'] = defaultdict(list) def modify_doc_targets(target, source, env): docspaces = env['TSDOC_DOCSPACES'] def variant_tgt(entry): # header files are located in the global directory (include/) # however, emitter will get an absolute path # so, make it again relative and put targets into build dir name = str(Dir('#').rel_path(entry)) name = env.BuildDir(PathJoin('tsdoc', name)) # XXX: SCons do not deletes old targets from in-memory fs
def build_prog(prog_name, sources, with_libs=None, *args, **kwargs): """Customized program builder. @param prog_name Program name @param sources Source file (or list of source files) @param with_libs Library name (or list of library names) to link with. @param install Binary flag to override default value from closure (`default_install`). """ if _ENABLE_DEBUG_PROG and self._env['VERBOSE'] is True: print(self._env.Dump()) # log_warn('LINKFLAGS:', self._env['LINKFLAGS'], '.') dump_info('build_prog:args', *args) dump_info('build_prog:kwargs', **kwargs) # Make sure sources is a list sources = listify(sources) + self._env['COMMON_OBJECTS'] install_flag = kwargs.pop('install', default_install) # Extract optional keywords arguments that we might extend cpp_paths = listify(kwargs.pop('CPPPATH', None)) ext_libs = listify(kwargs.pop('LIBS', None)) lib_paths = listify(kwargs.pop('LIBPATH', None)) # Process library dependencies - add libs specified in `with_libs` for lib_name in listify(with_libs): lib_keys = listify(self._get_matching_lib_keys(lib_name)) if len(lib_keys) == 1: # Matched internal library lib_key = lib_keys[0] # Extend prog sources with library nodes sources.extend(self._libs[lib_key]) elif len(lib_keys) > 1: # Matched multiple internal libraries - probably bad! raise StopError('Library identifier "{}" matched {} ' 'libraries ({}). Please use a fully ' 'qualified identifier instead!'.format( lib_name, len(lib_keys), ', '.join(lib_keys))) else: # empty lib_keys # Maybe it's an external library ext_lib = self._get_external_library(lib_name) if ext_lib: # Matched external library - extend target parameters cpp_paths.extend(ext_lib.cpp_paths) ext_libs.extend(ext_lib.libs) lib_paths.extend(ext_lib.lib_paths) else: raise StopError( 'Library identifier "{}" didn\'t match ' 'any library. Is it a typo?'.format(lib_name)) # Return extended construction environment parameters to kwargs if cpp_paths: kwargs['CPPPATH'] = cpp_paths if ext_libs: kwargs['LIBS'] = ext_libs if lib_paths: kwargs['LIBPATH'] = lib_paths # Build the program and add to prog nodes dict if installable prog_nodes = self._env.Program(prog_name, sources, *args, **kwargs) if install_flag: # storing each installable node in a dictionary instead of # defining InstallAs target on the spot, because there's # an "active" variant dir directive messing with paths. self._progs[module].extend(prog_nodes) return prog_nodes
def exists(env): try: import jinja2 except ImportError as e: raise StopError(ImportError, e.message)
def generate(env): if not exists(env): raise StopError("Cannot find grpc/protobuf compiler") env.SetDefault(PROTOGENDIR="generated")
return ret conf.AddTests({'CheckBinary': CheckBinary, 'CheckDesignatedInitializers': CheckDesignatedInitializers, 'CheckGCCSyncBuiltins': CheckGCCSyncBuiltins, 'CheckGCCAtomicBuiltins': CheckGCCAtomicBuiltins, 'CheckUnalignedMemAccess': CheckUnalignedMemAccess, 'CheckDladmOpen': CheckDladmOpen, 'CheckVsprintfSupportsCounting': CheckVsprintfSupportsCounting, 'CheckZpoolVdevName': CheckZpoolVdevName}) #------------------------------------------- # C compiler and standard library checks if not conf.CheckHeader('stdarg.h'): raise StopError('stdarg.h was not found!') # long long is used widely across agent code if not conf.CheckType('long long'): raise StopError("Your compiler doesn't support 'long long', abort!") # int64_t is used by ts_time_t if not conf.CheckType('int64_t', '#include <stdint.h>\n'): raise StopError("Your compiler doesn't support 'int64_t', abort!") conf.CheckDesignatedInitializers() conf.CheckDeclaration('min', '#include <stdlib.h>') conf.CheckDeclaration('max', '#include <stdlib.h>') conf.CheckDeclaration('round', '#include <math.h>')
def get_base_env(*args, **kwargs): """Initialize and return a base construction environment. All args received are passed transparently to SCons Environment init. """ # Initialize new construction environment env = Environment(*args, **kwargs) # pylint: disable=undefined-variable # If a flavor is activated in the external environment - use it if 'BUILD_FLAVOR' in os.environ: active_flavor = os.environ['BUILD_FLAVOR'] if not active_flavor in flavors(): raise StopError( '{} (from env) is not a known flavor.'.format(active_flavor)) log_warn('Using active flavor "{}" from your environment'.format( active_flavor)) env.flavors = [active_flavor] else: # If specific flavor target specified, skip processing other flavors # Otherwise, include all known flavors env.flavors = ( set(flavors()).intersection(COMMAND_LINE_TARGETS) # pylint: disable=undefined-variable or flavors()) # log_warn('flavors') # print(env.flavors) # Perform base construction environment customizations from site_config if '_common' in ENV_OVERRIDES: # https://docs.python.org/3/tutorial/controlflow.html#unpacking-argument-lists # env.Replace(ENV={'PATH': os.environ['PATH']}) env.Replace(**ENV_OVERRIDES['_common']) if '_common' in ENV_EXTENSIONS: env.Append(**ENV_EXTENSIONS['_common']) env.Append( BUILDERS={ 'LST': Builder(generator=lst_generator, suffix='.lst', src_suffix='.elf'), 'SIZ': Builder(generator=siz_generator, suffix='.siz', src_suffix='.elf'), 'BIN': Builder(generator=bin_generator, suffix='.bin', src_suffix='.elf'), 'HEX': Builder(generator=hex_generator, suffix='.hex', src_suffix='.elf') }) env.Append( BUILDERS={ 'ZBIN': Builder(action=zbin_generator, suffix='.bin.gz', src_suffix='.bin'), 'IMG': Builder(generator=img_generator, suffix='.img', src_suffix='.bin'), 'ZIMG': Builder(generator=zimg_generator, suffix='_gz.img', src_suffix=['.bin.gz', '.bin']), 'FLS': Builder(generator=fls_generator, suffix='.fls', src_suffix='.bin') }) # log_warn('base_Environment') # print(env.Dump()) log_warn('COMMAND_LINE_TARGETS: {}'.format(COMMAND_LINE_TARGETS)) log_warn('DEFAULT_TARGETS: {}'.format(DEFAULT_TARGETS)) log_warn('BUILD_TARGETS: {}'.format(BUILD_TARGETS)) return env