def generatedTestMethod(self): test_dir = util.writeTestFiles(filesForReporterTest(**kwargs), test_dir=self.test_dir) # build first, to make test timing more accurate: stdout, stderr, statuscode = cli.run(['--target', systemDefaultTarget(), 'build'], cwd=test_dir) #print('build:', stdout) #print('build:', stderr) #print('build statuscode was:', statuscode) self.assertEqual(statuscode, 0) tstart = time.time() stdout, stderr, statuscode = cli.run(['-vvv', '--target', systemDefaultTarget(), 'test'], cwd=test_dir) duration = time.time() - tstart # useful output for debugging failed tests: if bool(statuscode) == bool(kwargs['test_passes']) or \ duration >= 5.5 + kwargs['reporter_waits'] or \ (kwargs['test_speed'] == 'fast' and (duration >= 1.5 + kwargs['reporter_waits'])): print(stdout + stderr) print(statuscode) print('duration:', duration) if kwargs['test_passes']: self.assertEqual(statuscode, 0) else: self.assertNotEqual(statuscode, 0) # **no** tests should cause a timeout (Which is set at 4.5 seconds in # the test reporter), + the wait-for duration (+ 1 second slack for # process startup etc.) self.assertTrue(duration < 5.5 + kwargs['reporter_waits']) # if a test isn't slow, then it should run in less than 1 seconds if kwargs['test_speed'] == 'fast': self.assertTrue(duration < 1.5 + kwargs['reporter_waits'])
def test_tests(self): test_dir = util.writeTestFiles(Test_Tests, True) output = self.runCheckCommand(['--target', systemDefaultTarget(), 'build'], test_dir) output = self.runCheckCommand(['--target', systemDefaultTarget(), 'test'], test_dir) self.assertIn('test-a passed', output) self.assertIn('test-c passed', output) self.assertIn('test-d passed', output) self.assertIn('test-e passed', output) self.assertIn('test-f passed', output) self.assertIn('test-g passed', output) util.rmRf(test_dir)
def test_buildTests(self): test_dir = self.writeTestFiles(Test_Tests, True) stdout = self.runCheckCommand(['--target', systemDefaultTarget(), 'build'], test_dir) stdout = self.runCheckCommand(['--target', systemDefaultTarget(), 'test'], test_dir) self.assertIn('test-a', stdout) self.assertIn('test-c', stdout) self.assertIn('test-d', stdout) self.assertIn('test-e', stdout) self.assertIn('test-f', stdout) self.assertIn('test-g', stdout) rmRf(test_dir)
def test_testOutputFilterNotFound(self): test_dir = util.writeTestFiles(Test_Fitler_NotFound, True) stdout, stderr, statuscode = cli.run(['--target', systemDefaultTarget(), 'test'], cwd=test_dir) if statuscode == 0: print(stdout) print(stderr) self.assertNotEqual(statuscode, 0) util.rmRf(test_dir)
def test_buildInfo(self): test_dir = self.writeTestFiles(Test_Build_Info, True) # commit all the test files to git so that the VCS build info gets # defined: subprocess.check_call(['git', 'init', '-q'], cwd=test_dir) subprocess.check_call(['git', 'add', '.'], cwd=test_dir) subprocess.check_call(['git', 'commit', '-m', 'test build info automated commit', '-q'], cwd=test_dir) self.runCheckCommand(['--target', systemDefaultTarget(), 'build'], test_dir) build_time = datetime.datetime.utcnow() output = subprocess.check_output(['./build/' + systemDefaultTarget().split(',')[0] + '/source/test-trivial-exe'], cwd=test_dir).decode() self.assertIn('vcs clean: 1', output) # check build timestamp self.assertIn('build timestamp: ', output) build_timestamp_s = re.search('build timestamp: (.*)\n', output) self.assertTrue(build_timestamp_s) build_timestamp_s = build_timestamp_s.group(1) build_time_parsed = datetime.datetime.strptime(build_timestamp_s, '%Y-%m-%d-%H-%M-%S') build_time_skew = build_time_parsed - build_time self.assertTrue(abs(build_time_skew.total_seconds()) < 3)
def generatedTestMethod(self): test_dir = util.writeTestFiles(filesForReporterTest(**kwargs), test_dir=self.test_dir) # build first, to make test timing more accurate: stdout, stderr, statuscode = cli.run( ['--target', systemDefaultTarget(), 'build'], cwd=test_dir) #print('build:', stdout) #print('build:', stderr) #print('build statuscode was:', statuscode) self.assertEqual(statuscode, 0) tstart = time.time() stdout, stderr, statuscode = cli.run( ['-vvv', '--target', systemDefaultTarget(), 'test'], cwd=test_dir) duration = time.time() - tstart # useful output for debugging failed tests: if bool(statuscode) == bool(kwargs['test_passes']) or \ duration >= 5.5 + kwargs['reporter_waits'] or \ (kwargs['test_speed'] == 'fast' and (duration >= 1.5 + kwargs['reporter_waits'])): print(stdout + stderr) print(statuscode) print('duration:', duration) if kwargs['test_passes']: self.assertEqual(statuscode, 0) else: self.assertNotEqual(statuscode, 0) # **no** tests should cause a timeout (Which is set at 4.5 seconds in # the test reporter), + the wait-for duration (+ 1 second slack for # process startup etc.) self.assertTrue(duration < 5.5 + kwargs['reporter_waits']) # if a test isn't slow, then it should run in less than 1 seconds if kwargs['test_speed'] == 'fast': self.assertTrue(duration < 1.5 + kwargs['reporter_waits'])
def test_testOutputFilterFailing(self): test_dir = util.writeTestFiles(Test_Fitler_Fail, True) stdout, stderr, statuscode = cli.run(['--target', systemDefaultTarget(), 'test'], cwd=test_dir) if statuscode == 0: print(stdout) print(stderr) self.assertIn('test-a failed', '%s %s' % (stdout, stderr)) self.assertIn('test-c failed', '%s %s' % (stdout, stderr)) self.assertIn('test-d failed', '%s %s' % (stdout, stderr)) self.assertIn('test-e failed', '%s %s' % (stdout, stderr)) self.assertIn('test-f failed', '%s %s' % (stdout, stderr)) self.assertIn('test-g failed', '%s %s' % (stdout, stderr)) self.assertNotEqual(statuscode, 0) util.rmRf(test_dir)
def test_testOutputFilterFailing(self): test_dir = self.writeTestFiles(Test_Fitler_Fail, True) stdout, stderr, statuscode = cli.run(['--target', systemDefaultTarget(), 'test'], cwd=test_dir) if statuscode == 0: print(stdout) print(stderr) self.assertIn('test-a failed', '%s %s' % (stdout, stderr)) self.assertIn('test-c failed', '%s %s' % (stdout, stderr)) self.assertIn('test-d failed', '%s %s' % (stdout, stderr)) self.assertIn('test-e failed', '%s %s' % (stdout, stderr)) self.assertIn('test-f failed', '%s %s' % (stdout, stderr)) self.assertIn('test-g failed', '%s %s' % (stdout, stderr)) self.assertNotEqual(statuscode, 0) rmRf(test_dir)
def test_testAdditionalCMake(self): test_dir = util.writeTestFiles(util.Test_Test_Extra_CMake, True) output = self.runCheckCommand(['--target', systemDefaultTarget(), 'test'], test_dir) self.assertIn('test-trivial-lib-test-main passed', output) util.rmRf(test_dir)
def test_testOutputFilterPassing(self): test_dir = util.writeTestFiles(Test_Fitler_Pass, True) stdout = self.runCheckCommand(['--target', systemDefaultTarget(), 'test'], test_dir) util.rmRf(test_dir)
def nativeTarget(): assert(canBuildNatively()) return systemDefaultTarget()
def test_test(self): stdout = self.runCheckCommand(["--target", systemDefaultTarget(), "clean"], self.test_dir) stdout = self.runCheckCommand(["--target", systemDefaultTarget(), "test"], self.test_dir) self.assertNotIn("ignoredbyfname", stdout) self.assertNotIn("someothertest", stdout) self.assertNotIn("sometest", stdout)
def test_buildComplexSpaceInPath(self): test_dir = self.writeTestFiles(Test_Complex, True) stdout = self.runCheckCommand(['--target', systemDefaultTarget(), 'build'], test_dir) rmRf(test_dir)
def test_buildTrivialExe(self): test_dir = self.writeTestFiles(Test_Trivial_Exe) stdout = self.runCheckCommand(['--target', systemDefaultTarget(), 'build'], test_dir) rmRf(test_dir)
def test_testAdditionalCMake(self): test_dir = util.writeTestFiles(util.Test_Test_Extra_CMake, True) output = self.runCheckCommand( ['--target', systemDefaultTarget(), 'test'], test_dir) self.assertIn('test-trivial-lib-test-main passed', output) util.rmRf(test_dir)
def test_testOutputFilterPassing(self): test_dir = self.writeTestFiles(Test_Fitler_Pass, True) stdout = self.runCheckCommand( ['--target', systemDefaultTarget(), 'test'], test_dir) rmRf(test_dir)
def test_test(self): stdout = self.runCheckCommand(['--target', systemDefaultTarget(), 'clean'], self.test_dir) stdout = self.runCheckCommand(['--target', systemDefaultTarget(), 'test'], self.test_dir) self.assertNotIn('ignoredbyfname', stdout) self.assertNotIn('someothertest', stdout) self.assertNotIn('sometest', stdout)