def load( commit, target, test ): """ Load test result raises: IOError: When the results files is removed while reading. returns: TestResult instance """ commit = git.describe( commit ) fpath = os.path.join( config.STORAGE_DIR( test, commit=commit ), 'result.json' ) try: with open( fpath ) as results: data = json.load( results ) result = TestResult.unserialize( test, data ) result.test = test except IOError as e: # if the file did not exist, advice the user to run the test first if not os.path.exists( fpath ): raise ValueError( str( e )) else: raise logger.debug( "Loaded testresult: %s (commit: %s)" % ( str( result ), commit )) return result
def load(commit, target, test): """ Load test result raises: IOError: When the results files is removed while reading. returns: TestResult instance """ commit = git.describe(commit) fpath = os.path.join(config.STORAGE_DIR(test, commit=commit), 'result.json') try: with open(fpath) as results: data = json.load(results) result = TestResult.unserialize(test, data) result.test = test except IOError as e: # if the file did not exist, advice the user to run the test first if not os.path.exists(fpath): raise ValueError(str(e)) else: raise logger.debug("Loaded testresult: %s (commit: %s)" % (str(result), commit)) return result
def plugin_environ(test=None): """ extend a copy of os.environ with some testing environ vars. these include the config.ENV names. kwargs: test {Test}: for setting optional test specific variables (TEST_NAME, TEST_LOG) return: environment {dict} """ env = os.environ.copy() env['TEMP_DIR'] = config.TEMP_DIR env['TESTS_DIR'] = config.TESTS_DIR env['SRC_DIR'] = config.SRC_DIR for key, value in config.ENV.items(): env[key] = str(value) if test is not None: env['TEST_NAME'] = test.name env['TEST_LOG'] = test.log_path() env['RESULTS_DIR'] = os.path.join( config.TEMP_DIR, config.STORAGE_DIR(test, commit=git.describe())) return env
def plugin_environ( test=None ): """ extend a copy of os.environ with some testing environ vars. these include the config.ENV names. kwargs: test {Test}: for setting optional test specific variables (TEST_NAME, TEST_LOG) return: environment {dict} """ env = os.environ.copy() env[ 'TEMP_DIR' ] = config.TEMP_DIR env[ 'TESTS_DIR' ] = config.TESTS_DIR env[ 'SRC_DIR' ] = config.SRC_DIR for key, value in config.ENV.items(): env[ key ] = str( value ) if test is not None: env[ 'TEST_NAME' ] = test.name env[ 'TEST_LOG' ] = test.log_path() env[ 'RESULTS_DIR' ] = os.path.join( config.TEMP_DIR, config.STORAGE_DIR( test, commit=git.describe() ) ) return env
def exists( commit, target, test ): """ Check if a test results exists returns: bool """ commit = git.describe( commit ) fpath = config.STORAGE_DIR( test, commit=commit ) return os.path.exists( fpath )
def exists(commit, target, test): """ Check if a test results exists returns: bool """ commit = git.describe(commit) fpath = config.STORAGE_DIR(test, commit=commit) return os.path.exists(fpath)
def __init__( self, test, start, stop, target, commit=None ): assert start is not None assert stop is not None self.test = test self.started = start self.completed = stop self.target = target self.commit = commit or git.describe() self.metrics = {} # additional test result files by name self.files = {}
def discover_tests( args ): tests = [] # try to find the suites and append the testnames # of the suite for name in args.suite: logger.info( "Loading tests from suite `%s`" % name ) # make sure to have a valid test suite name try: suite = config.SUITES[ name ] for descr in suite: for fname in Test.glob( descr ): logger.debug( "Adding test `%s` to tests." % fname ) tests.append( Test( fname )) except KeyError: logger.error( "We looked, but a test suite with name `%s` was not found." % name ) # queue the named tests for names in [ Test.glob( name ) for name in args.tests ]: for name in names: tests.append( Test( name )) # expand excludes using globbing excludes = [] for ex in args.exclude: excludes += Test.glob( ex ) # unqueue excluded tests tests = [ t for t in tests if t.name not in excludes ] # unqueue tests that already have results # if complement option is given if args.complement: targets = discover_targets( args ) commit = args.commit or git.describe() # assume tested filtered = [] for test in tests: tested = True for t in targets: if not StorageProvider.exists( commit, t, test ): tested = False if not tested: filtered.append( test ) tests = filtered return tests
def run( self, target=config.DEFAULT_TARGET, commit=None, store=False ): # make sure we have work to do assert len( self.tests ) != 0, "No tests to run." # if a commit is given, we need to checkout out the commit # but keep the tests directory at the current branch try: # create a lock file to prevent multiple runners on the same repo with open( config.LOCK_FILE, 'w' ) as fh: fh.write( str( datetime.datetime.now() )) # checkout the correct commit if commit is not None: current = git.current_branch() # if not on any branch currently # remember the current commit if current == "HEAD": current = git.describe() logger.warning( "Checking out commit `%s`" % commit ) git.checkout( commit ) git.checkout( current, paths=[ config.TESTS_DIR ]) # actual running of the tests self._run_tests( target=target, store=store ) # if a git error is caught while checking out the commit on # which to run the tests # we can't do anything except git.GitError as e: logger.error( "Could not checkout `%s`... Sorry!" % commit ) raise finally: # make sure to always checkout the original commit again if commit is not None: try: git.checkout( current ) logger.info( "Checked out the original HEAD again!" ) except git.GitError as e: logger.error( "Could not checkout original branch... you'll have to do that yourself." ) raise # make sure to remove the lock file io.remove_file( config.LOCK_FILE )
def load_results( self, commit=git.describe(), targets=[ config.DEFAULT_TARGET ], tests=[] ): """ Load results based on the passed commit, targets and tests """ # append new targets to the target list self.targets += [ t for t in targets if t not in self.targets ] for target in targets: config.set_target( target ) for test in tests: try: self.add_result( JsonStorageProvider.load( commit, target, test )) logger.debug( "Found result: %s, %s, %s" % ( commit, target, test )) except ValueError as e: logger.error( "No test results exists yet for test `%s`" % test.name )
def load_results(self, commit=git.describe(), targets=[config.DEFAULT_TARGET], tests=[]): """ Load results based on the passed commit, targets and tests """ # append new targets to the target list self.targets += [t for t in targets if t not in self.targets] for target in targets: config.set_target(target) for test in tests: try: self.add_result( JsonStorageProvider.load(commit, target, test)) logger.debug("Found result: %s, %s, %s" % (commit, target, test)) except ValueError as e: logger.error("No test results exists yet for test `%s`" % test.name)