def load( commit, target, test ): """ Load test result raises: IOError: When the results files is removed while reading. returns: TestResult instance """ commit = git.describe( commit ) fpath = os.path.join( config.STORAGE_DIR( test, commit=commit ), 'result.json' ) try: with open( fpath ) as results: data = json.load( results ) result = TestResult.unserialize( test, data ) result.test = test except IOError as e: # if the file did not exist, advice the user to run the test first if not os.path.exists( fpath ): raise ValueError( str( e )) else: raise logger.debug( "Loaded testresult: %s (commit: %s)" % ( str( result ), commit )) return result
def load(commit, target, test):
""" Load test result
raises:
IOError: When the results files is removed while reading.
returns:
TestResult instance
"""
commit = git.describe(commit)
fpath = os.path.join(config.STORAGE_DIR(test, commit=commit),
'result.json')
try:
with open(fpath) as results:
data = json.load(results)
result = TestResult.unserialize(test, data)
result.test = test
except IOError as e:
# if the file did not exist, advice the user to run the test first
if not os.path.exists(fpath):
raise ValueError(str(e))
else:
raise
logger.debug("Loaded testresult: %s (commit: %s)" %
(str(result), commit))
return result
def plugin_environ(test=None):
""" extend a copy of os.environ with some testing environ vars.
these include the config.ENV names.
kwargs:
test {Test}:
for setting optional test specific variables
(TEST_NAME, TEST_LOG)
return:
environment {dict}
"""
env = os.environ.copy()
env['TEMP_DIR'] = config.TEMP_DIR
env['TESTS_DIR'] = config.TESTS_DIR
env['SRC_DIR'] = config.SRC_DIR
for key, value in config.ENV.items():
env[key] = str(value)
if test is not None:
env['TEST_NAME'] = test.name
env['TEST_LOG'] = test.log_path()
env['RESULTS_DIR'] = os.path.join(
config.TEMP_DIR, config.STORAGE_DIR(test, commit=git.describe()))
return env
def plugin_environ( test=None ):
""" extend a copy of os.environ with some testing environ vars.
these include the config.ENV names.
kwargs:
test {Test}:
for setting optional test specific variables
(TEST_NAME, TEST_LOG)
return:
environment {dict}
"""
env = os.environ.copy()
env[ 'TEMP_DIR' ] = config.TEMP_DIR
env[ 'TESTS_DIR' ] = config.TESTS_DIR
env[ 'SRC_DIR' ] = config.SRC_DIR
for key, value in config.ENV.items():
env[ key ] = str( value )
if test is not None:
env[ 'TEST_NAME' ] = test.name
env[ 'TEST_LOG' ] = test.log_path()
env[ 'RESULTS_DIR' ] = os.path.join(
config.TEMP_DIR,
config.STORAGE_DIR( test, commit=git.describe() )
)
return env
def exists( commit, target, test ): """ Check if a test results exists returns: bool """ commit = git.describe( commit ) fpath = config.STORAGE_DIR( test, commit=commit ) return os.path.exists( fpath )
def exists(commit, target, test):
""" Check if a test results exists
returns:
bool
"""
commit = git.describe(commit)
fpath = config.STORAGE_DIR(test, commit=commit)
return os.path.exists(fpath)
def __init__( self, test, start, stop, target, commit=None ):
assert start is not None
assert stop is not None
self.test = test
self.started = start
self.completed = stop
self.target = target
self.commit = commit or git.describe()
self.metrics = {}
# additional test result files by name
self.files = {}
def discover_tests( args ): tests = [] # try to find the suites and append the testnames # of the suite for name in args.suite: logger.info( "Loading tests from suite `%s`" % name ) # make sure to have a valid test suite name try: suite = config.SUITES[ name ] for descr in suite: for fname in Test.glob( descr ): logger.debug( "Adding test `%s` to tests." % fname ) tests.append( Test( fname )) except KeyError: logger.error( "We looked, but a test suite with name `%s` was not found." % name ) # queue the named tests for names in [ Test.glob( name ) for name in args.tests ]: for name in names: tests.append( Test( name )) # expand excludes using globbing excludes = [] for ex in args.exclude: excludes += Test.glob( ex ) # unqueue excluded tests tests = [ t for t in tests if t.name not in excludes ] # unqueue tests that already have results # if complement option is given if args.complement: targets = discover_targets( args ) commit = args.commit or git.describe() # assume tested filtered = [] for test in tests: tested = True for t in targets: if not StorageProvider.exists( commit, t, test ): tested = False if not tested: filtered.append( test ) tests = filtered return tests
def run( self, target=config.DEFAULT_TARGET, commit=None, store=False ): # make sure we have work to do assert len( self.tests ) != 0, "No tests to run." # if a commit is given, we need to checkout out the commit # but keep the tests directory at the current branch try: # create a lock file to prevent multiple runners on the same repo with open( config.LOCK_FILE, 'w' ) as fh: fh.write( str( datetime.datetime.now() )) # checkout the correct commit if commit is not None: current = git.current_branch() # if not on any branch currently # remember the current commit if current == "HEAD": current = git.describe() logger.warning( "Checking out commit `%s`" % commit ) git.checkout( commit ) git.checkout( current, paths=[ config.TESTS_DIR ]) # actual running of the tests self._run_tests( target=target, store=store ) # if a git error is caught while checking out the commit on # which to run the tests # we can't do anything except git.GitError as e: logger.error( "Could not checkout `%s`... Sorry!" % commit ) raise finally: # make sure to always checkout the original commit again if commit is not None: try: git.checkout( current ) logger.info( "Checked out the original HEAD again!" ) except git.GitError as e: logger.error( "Could not checkout original branch... you'll have to do that yourself." ) raise # make sure to remove the lock file io.remove_file( config.LOCK_FILE )
def load_results( self, commit=git.describe(), targets=[ config.DEFAULT_TARGET ], tests=[] ): """ Load results based on the passed commit, targets and tests """ # append new targets to the target list self.targets += [ t for t in targets if t not in self.targets ] for target in targets: config.set_target( target ) for test in tests: try: self.add_result( JsonStorageProvider.load( commit, target, test )) logger.debug( "Found result: %s, %s, %s" % ( commit, target, test )) except ValueError as e: logger.error( "No test results exists yet for test `%s`" % test.name )
def load_results(self,
commit=git.describe(),
targets=[config.DEFAULT_TARGET],
tests=[]):
""" Load results based on the passed commit, targets and tests
"""
# append new targets to the target list
self.targets += [t for t in targets if t not in self.targets]
for target in targets:
config.set_target(target)
for test in tests:
try:
self.add_result(
JsonStorageProvider.load(commit, target, test))
logger.debug("Found result: %s, %s, %s" %
(commit, target, test))
except ValueError as e:
logger.error("No test results exists yet for test `%s`" %
test.name)
