def test_resolve_path(): assert resolve_path("$HOME") assert resolve_path("~") random_name = "".join(random.choice(string.ascii_letters) for i in range(10)) # test a directory path that doesn't exist in $HOME with random key, but setting exist=False will return # path but doesn't mean file exists path = resolve_path(os.path.join("$HOME", random_name), exist=False) # checking if path is not file, or directory and not None. This is only valid when exist=False is set assert not is_file(path) assert not is_dir(path) assert path is not None
def load_paths(self): """Add all paths to search for buildspecs. We must read configuration file and check property ``buildspec_roots`` for list of directories to search. We check all directories exist, if any fail we don't add them to path. In addition, we add the default buildspec path where we find tutorials and general tests. """ buildspec_paths = self.configuration.target_config.get( "buildspec_roots") or [] if buildspec_paths: self.roots += buildspec_paths # only load default buildspecs if 'load_default_buildspecs' set to True if self.configuration.target_config.get("load_default_buildspecs"): self.paths += BUILDSPEC_DEFAULT_PATH # for every root buildspec defined in configuration or via --root option, # we resolve path and if path exist add to self.paths. The path must be a # directory. If its file, we ignore it if self.roots: for root in self.roots: path = resolve_path(root, exist=False) if not os.path.exists(path): print(f"Path: {path} does not exist!") if is_file(path): print(f"Path: {path} must be a directory not a file") if is_dir(path): self.paths.append(path)
def _resolve_source(self): """This method resolves full path to source file, it checks for absolute path first before checking relative path that is relative to Buildspec recipe. """ # attempt to resolve path based on 'source' field. One can specify an absolute path if specified we honor it self.abspath_sourcefile = resolve_path(self.sourcefile) # One can specify a relative path to where buildspec is located when using 'source' field so we try again if not self.abspath_sourcefile: self.abspath_sourcefile = resolve_path( os.path.join(os.path.dirname(self.buildspec), self.sourcefile)) # raise error if we can't find source file to compile if not self.abspath_sourcefile: raise BuildTestError( f"Failed to resolve path specified in field 'source': {self.sourcefile}" )
def resolve_testdirectory(buildtest_configuration, cli_testdir=None): """This method resolves which test directory to select. For example, one can specify test directory via command line ``buildtest build --testdir <path>`` or path in configuration file. The default is $BUILDTEST_ROOT/var/tests :param buildtest_configuration: loaded buildtest configuration as a dict. :type buildtest_configuration: dict :param cli_testdir: test directory from command line ``buildtest build --testdir`` :type cli_testdir: str :return: Path to test directory to use :rtype: str """ prefix = buildtest_configuration.get("testdir") # variable to set test directory if prefix is set prefix_testdir = None if prefix: prefix = resolve_path(prefix, exist=False) if prefix: prefix_testdir = prefix if cli_testdir: # resolve full path for test directory specified by --testdir option cli_testdir = resolve_path(cli_testdir, exist=False) # Order of precedence when detecting test directory # 1. Command line option --testdir # 2. Configuration option specified by 'testdir' # 3. Defaults to $BUILDTEST_ROOT/var/tests test_directory = (cli_testdir or prefix_testdir or os.path.join(BUILDTEST_ROOT, "var", "tests")) if not test_directory: raise BuildTestError( "Invalid value for test directory, please specify a valid directory path through command line (--testdir) or configuration file" ) create_dir(test_directory) return test_directory
def resolve_settings_file(settings_path=None): """Returns path to buildtest settings file that should be used. If there is a user defined buildtest settings ($HOME/.buildtest/config.yml) it will be honored, otherwise default settings from buildtest will be used. """ if settings_path: settings_path = resolve_path(settings_path) return settings_path # if buildtest settings file exist return it otherwise return default file if os.path.exists(USER_SETTINGS_FILE): return USER_SETTINGS_FILE return DEFAULT_SETTINGS_FILE
def __init__(self, buildspec, buildexecutor): """The init method will run some checks against buildspec before loading buildspec. We retrieve available schemas via method ``get_schemas_available`` and check if ``type`` in buildspec match available schema. We validate the entire buildspec with global.schema.json and validate each test section with the designated type schema. If there is any error during the init method, an exception will be raised. :param buildspec: the pull path to the Buildspec file, must exist. :type buildspec: str, required :param buildexecutor: an instance of BuildExecutor class defines Executors from configuration file :type buildexecutor: BuildExecutor, required """ self.logger = logging.getLogger(__name__) if not isinstance(buildexecutor, BuildExecutor): raise BuildTestError( "Invalid type argument for 'buildexecutor', must be of type BuildExecutor" ) self.buildexecutors = buildexecutor.list_executors() # if invalid input for buildspec if not buildspec: raise BuildTestError( "Invalid input type for Buildspec, must be of type 'string'.") self.buildspec = resolve_path(buildspec) if not self.buildspec: raise BuildTestError("There is no file named: %s " % buildspec) if is_dir(self.buildspec): raise BuildTestError( f"Detected {self.buildspec} is a directory, please provide a file path (not a directory path) to BuildspecParser." ) self.recipe = load_recipe(self.buildspec) # ensure self.recipe exists after loading recipe assert self.recipe # validate each schema defined in the recipes self._validate()
def filter_buildspecs_from_report(self): # filter_buildspecs = self.report.keys() # print (self.filter, filter_buildspecs) # This section filters the buildspec, if its invalid file or not found in cache # we raise error, otherwise we set filter_buildspecs to the filter argument 'buildspec' # by default all keys from report are buildspec files to process self.filtered_buildspecs = self.report.keys() # if --filter option not specified we return from method if not self.filter: return if self.filter.get("buildspec"): # resolve path for buildspec filter key, its possible if file doesn't exist method returns None resolved_buildspecs = resolve_path(self.filter["buildspec"]) # if file doesn't exist we terminate with message if not resolved_buildspecs: print( f"Invalid File Path for filter field 'buildspec': {self.filter['buildspec']}" ) sys.exit(0) # if file not found in cache we exit if not resolved_buildspecs in self.report.keys(): print( f"buildspec file: {resolved_buildspecs} not found in cache" ) sys.exit(0) # need to set as a list since we will loop over all tests self.filtered_buildspecs = [resolved_buildspecs] # ensure 'state' field in filter is either 'PASS' or 'FAIL', if not raise error if self.filter.get("state"): if self.filter["state"] not in ["PASS", "FAIL"]: print( f"filter argument 'state' must be 'PASS' or 'FAIL' got value {self.filter['state']}" ) sys.exit(0)
def main(): """Entry point to buildtest.""" if not os.getenv("BUILDTEST_COLOR"): os.environ["BUILDTEST_COLOR"] = "True" # create a temporary file to store logfile and we don't delete file by setting 'delete=False' # by default tempfile will delete file upon exit. tf = tempfile.NamedTemporaryFile(prefix="buildtest_", delete=False, suffix=".log") dest_logfile = tf.name logger = init_logfile(dest_logfile) logger.info("Starting buildtest log") create_dir(BUILDTEST_USER_HOME) create_dir(var_root) # Create a build test system, and check requirements # BuildTestSystem() system.check() parser = BuildTestParser() args = parser.parse_options() if args.debug: streamlog(args.debug) if args.subcommands == "build": # settings_file = resolve_settings_file(args.config) # check_settings(args.config) cmd = BuildTest( config_file=args.config, buildspecs=args.buildspec, exclude_buildspecs=args.exclude, executors=args.executor, tags=args.tags, filter_tags=args.filter_tags, rebuild=args.rebuild, stage=args.stage, testdir=args.testdir, ) cmd.build() logdir = buildtest_configuration.target_config.get("logdir") if not logdir: print(f"Writing Logfile to: {dest_logfile}") sys.exit(0) logdir = resolve_path(logdir, exist=False) if logdir: create_dir(logdir) fname = os.path.basename(dest_logfile) logpath = os.path.join(logdir, fname) shutil.copy2(dest_logfile, logpath) print(f"Writing Logfile to: {logpath}") else: print(f"Writing Logfile to: {dest_logfile}") # store copy of logfile at $BUILDTEST_ROOT/buildtest.log. A convenient location for user to # find logfile for last build, this will be overwritten for every subsequent build. shutil.copy2( dest_logfile, os.path.join(os.getenv("BUILDTEST_ROOT"), "buildtest.log")) return settings_file = resolve_settings_file() logger.info(f"Processing buildtest configuration file: {settings_file}") check_settings(settings_file) # implementation for 'buildtest buildspec find' if args.subcommands == "buildspec": buildspec_find(args=args, settings_file=settings_file) return if args.subcommands and args.func: args.func(args)
def discover_by_buildspecs(buildspec): """Given a buildspec file specified by the user with ``buildtest build --buildspec``, discover one or more files and return a list for buildtest to process. This method is called once per argument of ``--buildspec`` or ``--exclude`` option. If its a directory path we recursively find all buildspecs with .yml extension. If filepath doesn't exist or file extension is not .yml we return None and capture error in log. # file path buildtest build --buildspec tutorials/hello.sh.yml # directory path buildtest build --buildspec tutorials :param buildspec: Input argument from ``buildtest build --buildspec`` :type buildspec: str :return: A list of discovered buildspec with resolved path, if its invalid we return None :rtype: list or None """ buildspecs = [] # if buildspec doesn't exist print message and log error and return if not os.path.exists(os.path.abspath(buildspec)): msg = ( f"Unable to find any buildspecs with name: {os.path.abspath(buildspec)} " + "Please provide an absolute or relative path to a directory or file relative to current directory." ) print(msg) logger.error(msg) return # Now handle path based on being a directory or file path if os.path.isdir(buildspec): logger.debug( f"Buildspec File: {buildspec} is a directory so traversing directory tree to find all Buildspec files with .yml extension" ) buildspecs = walk_tree(buildspec, ".yml") elif os.path.isfile(buildspec): # if buildspec doesn't end in .yml extension we print message and return None if not re.search(".yml$", buildspec): msg = f"{buildspec} does not end in file extension .yml" print(msg) logger.error(msg) return buildspecs = [buildspec] logger.debug(f"BuildSpec: {buildspec} is a file") # If we don't have any files discovered if not buildspecs: msg = "No Buildspec files found with input: %s." % buildspec print(msg) logger.error(msg) return # return all buildspec by resolving path, this gets the real canonical path and address shell expansion and user expansion buildspecs = [resolve_path(file) for file in buildspecs] logger.info(f"Found the following config files: {buildspecs}") return buildspecs
def main(): """Entry point to buildtest.""" # create a temporary file to store logfile and we don't delete file by setting 'delete=False' # by default tempfile will delete file upon exit. tf = tempfile.NamedTemporaryFile(prefix="buildtest_", delete=False, suffix=".log") dest_logfile = tf.name logger = init_logfile(dest_logfile) logger.info("Starting buildtest log") create_dir(BUILDTEST_USER_HOME) create_dir(var_root) # Create a build test system, and check requirements # BuildTestSystem() system.check() parser = BuildTestParser() args = parser.parse_options() if args.debug: streamlog(args.debug) # invoking load_settings will attempt to initialize buildtest settings and # load the schema settings_file = resolve_settings_file() logger.info(f"Processing buildtest configuration file: {settings_file}") buildtest_configuration = check_settings(settings_file, retrieve_settings=True) if args.subcommands == "build": func_build_subcmd(args, buildtest_configuration) else: if args.subcommands and args.func: args.func(args) return logdir = buildtest_configuration.get("logdir") if not logdir: print(f"Writing Logfile to: {dest_logfile}") sys.exit(0) logdir = resolve_path(logdir, exist=False) if logdir: create_dir(logdir) fname = os.path.basename(dest_logfile) logpath = os.path.join(logdir, fname) shutil.copy2(dest_logfile, logpath) print(f"Writing Logfile to: {logpath}") else: print(f"Writing Logfile to: {dest_logfile}") # store copy of logfile at $BUILDTEST_ROOT/buildtest.log. A convenient location for user to # find logfile for last build, this will be overwritten for every subsequent build. shutil.copy2(dest_logfile, os.path.join(os.getenv("BUILDTEST_ROOT"), "buildtest.log"))