def test_clean_wait(self): """Test clean command after waiting for tests to finish.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args([ 'run', '-H', 'this', 'clean_test' ]) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) time.sleep(5) args = arg_parser.parse_args([ 'clean' ]) clean_cmd = commands.get_command(args.command_name) clean_cmd.outfile = StringIO() clean_cmd.errfile = StringIO() self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0)
def test_cancel_series_test(self): """Test cancel command with combination of series and tests.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args( ['run', '-H', 'this', 'hello_world.hello', 'hello_world.world']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) tests = [] series_id = series.TestSeries.load_user_series_id(self.pav_cfg) tests.append(series_id) tests.extend( series.TestSeries.from_id(self.pav_cfg, int(series_id[1:])).tests) args = arg_parser.parse_args( ['cancel', tests[0], str(tests[1]), str(tests[2])]) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = StringIO() cancel_cmd.errfile = StringIO() self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_clean_with_older_than_flag(self): """Test clean command with multiple date formats.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['clean', '--older-than', '5 weeks']) clean_cmd = commands.get_command(args.command_name) clean_cmd.outfile = StringIO() clean_cmd.errfile = StringIO() self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0) args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['clean', '--older-than', 'Jul 3 2019']) clean_cmd = commands.get_command(args.command_name) clean_cmd.outfile = StringIO() clean_cmd.errfile = StringIO() self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0)
def test_command_plugins(self): """Make sure command plugin loading is sane.""" # Get an empty pavilion config and set some config dirs on it. pav_cfg = config.PavilionConfigLoader().load_empty() # We're loading multiple directories of plugins - AT THE SAME TIME! pav_cfg.config_dirs = [ os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir'), os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir2') ] plugins.initialize_plugins(pav_cfg) commands.get_command('poof').run(pav_cfg, []) commands.get_command('blarg').run(pav_cfg, []) # Clean up our plugin initializations. plugins._reset_plugins() pav_cfg.config_dirs.append( os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir_conflicts')) self.assertRaises(plugins.PluginError, lambda: plugins.initialize_plugins(pav_cfg)) # Clean up our plugin initializations. plugins._reset_plugins()
def test_cancel_series_test(self): """Test cancel command with combination of series and tests.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args( ['run', '-H', 'this', 'cancel_test.test1', 'cancel_test.test2']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) tests = [] series_id = series.TestSeries.load_user_series_id(self.pav_cfg) tests.append(series_id) args = arg_parser.parse_args([ 'cancel', tests[0], ]) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = cancel_cmd.errfile = StringIO() self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_cancel_sched_check(self): """Cancel Test and make sure it is cancelled through scheduler.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this' 'hello_world2']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['cancel']) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = StringIO() cancel_cmd.errfile = StringIO() test = [] series_id = series.TestSeries.load_user_series_id(self.pav_cfg) test.append(series_id) test_list = [] test_list.extend( series.TestSeries.from_id(self.pav_cfg, int(test[0][1:])).tests) for test_id in test_list: test = PavTest.load(self.pav_cfg, test_id) if test.status.current().state != STATES.COMPLETE: sched = schedulers.get_scheduler_plugin(test.scheduler) sched_status = sched.job_status(self.pav_cfg, test) self.assertIn("SCHED_CANCELLED", str(sched_status))
def test_cancel_status_json(self): """Test cancel command with status flag and json flag.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args( ['run', '-H', 'this', 'hello_world.world']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['cancel', '-s', '-j']) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = StringIO() cancel_cmd.errfile = StringIO() self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0) results = cancel_cmd.outfile.getvalue().split('\n')[-1].strip().encode( 'UTF-8') results = results[4:].decode('UTF-8') results = json.loads(results) self.assertNotEqual(len(results), 0)
def test_multi_build(self): """Make sure we can build multiple simultanious builds on both the front-end and the nodes.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args([ 'run', '-H', 'this', 'build_parallel' ]) run_cmd = commands.get_command(args.command_name) # type: RunCommand run_ret = run_cmd.run(self.pav_cfg, args) run_cmd.outfile.seek(0) self.assertEqual(run_ret, 0, msg=run_cmd.outfile.read()) for test in run_cmd.last_tests: test.wait(timeout=4) # Make sure we actually built separate builds builds = [test.builder for test in run_cmd.last_tests] build_names = set([b.name for b in builds]) self.assertEqual(len(build_names), 4) for test in run_cmd.last_tests: self.assertEqual(test.results['result'], 'PASS', msg='Test {} status: {}' .format(test.id, test.status.current()))
def test_clean(self): """Test clean command with no arguments.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test']) run_cmd = commands.get_command(args.command_name) run_cmd.silence() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['clean']) clean_cmd = commands.get_command(args.command_name) clean_cmd.silence() self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0)
def test_status_history(self): # Testing that status works with history flag status_cmd = commands.get_command('status') out = io.StringIO() status_cmd.outfile = out parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) test = self._quick_test() raw = schedulers.get_plugin('raw') raw.schedule_test(self.pav_cfg, test) end = time.time() + 5 while test.check_run_complete() is None and time.time() < end: time.sleep(.1) args = parser.parse_args(['--history', str(test.id)]) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) out.seek(0) output = out.readlines()[4:] statuses = test.status.history() self.assertEqual(len(output), len(statuses)) for i in range(len(output)): self.assertTrue(statuses[i].state in output[i])
def test_local_builds_only(self): """Make sure we can just build multiple simultanious builds on both the front-end and the nodes.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args( ['build', '-H', 'this', '--local-builds-only', 'build_parallel']) build_cmd = commands.get_command(args.command_name) # type: RunCommand build_ret = build_cmd.run(self.pav_cfg, args) build_cmd.outfile.seek(0) self.assertEqual(build_ret, 0, msg=build_cmd.outfile.read()) for test in build_cmd.last_tests: test.wait(timeout=10) # Make sure we actually built separate builds builds = [test.builder for test in build_cmd.last_tests] build_names = set([b.name for b in builds]) self.assertEqual(len(build_names), 2) for test in build_cmd.last_tests: self.assertEqual(test.status.current().state, STATES.BUILD_DONE, msg='Test {} status: {}'.format( test.id, test.status.current()))
def test_status_command_with_sched(self): """Test status command when test is 'SCHEDULED'.""" cfg = file_format.TestConfigLoader().validate({ 'scheduler': 'raw', 'run': { 'env': { 'foo': 'bar', }, 'cmds': ['sleep 1'], }, }) cfg['name'] = 'testytest' test = self._quick_test(cfg, build=False, finalize=False) test.build() schedulers.get_plugin(test.scheduler) \ .schedule_test(self.pav_cfg, test) status_cmd = commands.get_command('status') status_cmd.outfile = io.StringIO() parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) args = parser.parse_args([str(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) args = parser.parse_args(['-j', str(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def test_build_parallel_lots(self): """Make sure building works beyond the parallel building limit.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args([ 'run', '-H', 'this', 'build_parallel_lots' ]) run_cmd = commands.get_command(args.command_name) # type: RunCommand run_ret = run_cmd.run(self.pav_cfg, args) run_cmd.outfile.seek(0) self.assertEqual(run_ret, 0, msg=run_cmd.outfile.read()) for test in run_cmd.last_tests: test.wait(timeout=5) # Make sure we actually built separate builds builds = [test.builder for test in run_cmd.last_tests] build_names = set([b.name for b in builds]) self.assertEqual(len(build_names), 8) for test in run_cmd.last_tests: self.assertEqual(test.results['result'], 'PASS', msg='Test {} status: {}' .format(test.id, test.status.current()))
def test_status_summary(self): # Testing that status works with summary flag status_cmd = commands.get_command('status') status_cmd.outfile = io.StringIO() parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) arg_list = ['-s'] args = parser.parse_args(arg_list) # Test that an empty working_dir fails correctly self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) base_cfg = self._quick_test_cfg() test_cfg1 = base_cfg.copy() test_cfg1['name'] = 'test1' test_cfg2 = base_cfg.copy() test_cfg2['name'] = 'test2' test_cfg3 = base_cfg.copy() test_cfg3['name'] = 'test3' configs = [test_cfg1, test_cfg2, test_cfg3] tests = [self._quick_test(cfg) for cfg in configs] for test in tests: test.RUN_SILENT_TIMEOUT = 1 # Testing that summary flags return correctly self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def test_multi_build_fail(self): """Make sure we can build multiple simultanious builds on both the front-end and the nodes.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args([ 'run', '-H', 'this', 'build_parallel_fail' ]) run_cmd = commands.get_command(args.command_name) # type: RunCommand self.assertEqual(run_cmd.run(self.pav_cfg, args), 22) # Make sure we actually built separate builds builds = [test.builder for test in run_cmd.last_tests] build_names = set([b.name for b in builds]) self.assertEqual(len(build_names), 4) statuses = [test.status.current().state for test in run_cmd.last_tests] statuses = set(statuses) self.assertEqual(statuses, {STATES.ABORTED, STATES.BUILD_FAILED}) self.assertTrue(all([test.complete for test in run_cmd.last_tests]))
def test_run_timeouts(self): """Make sure run timeout file works as expected.""" run_cmd = commands.get_command('run') run_cmd.silence() arg_parser = arguments.get_parser() # All test follow the same pattern seen above, but we can run them all # at once, since a run timeout doesn't effect the others. args = arg_parser.parse_args(['run', 'timeout_run_tests']) self.assertEqual(run_cmd.run(self.pav_cfg, args), 0) time.sleep(35) correct_statuses = { 'timeout_run_tests.GoodRun': 'COMPLETE', 'timeout_run_tests.GoodRun2': 'COMPLETE', 'timeout_run_tests.GoodRun3': 'COMPLETE', 'timeout_run_tests.BadRun': 'RUN_TIMEOUT', 'timeout_run_tests.BadRun2': 'RUN_TIMEOUT', 'timeout_run_tests.BadRun3': 'RUN_TIMEOUT' } status_args = arg_parser.parse_args(['status']) statuses = get_statuses(self.pav_cfg, status_args, io.StringIO()) for test_status in statuses: self.assertEqual(correct_statuses[test_status['name']], test_status['state'])
def test_sys_name_tracker(self): """Make sure the expected values are stored in the user.json file.""" user = utils.get_login() sys_vars = system_variables.get_vars(True) sys_name = sys_vars['sys_name'] arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', 'hello_world']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = io.StringIO() run_cmd.errfile = run_cmd.outfile run_cmd.run(self.pav_cfg, args) series = run_cmd.last_series json_file = self.pav_cfg.working_dir / 'users' json_file /= '{}.json'.format(user) with json_file.open('r') as json_series_file: data = json.load(json_series_file) self.assertEqual(data[sys_name], series.sid)
def test_log_arguments(self): log_cmd = commands.get_command('log') parser = argparse.ArgumentParser() log_cmd._setup_arguments(parser) # run a simple test test = self._quick_test(finalize=False) raw = schedulers.get_plugin('raw') raw.schedule_test(self.pav_cfg, test) state = test.status.current().state end = time.time() + 1 while ('ERROR' not in state and 'FAIL' not in state and state != STATES.COMPLETE and time.time() < end): time.sleep(.1) # test `pav log run test` args = parser.parse_args(['run', str(test.id)]) self.assertEqual(args.test, test.id) out = io.StringIO() err = io.StringIO() log_cmd.outfile = out log_cmd.errfile = err result = log_cmd.run(self.pav_cfg, args) err.seek(0) out.seek(0) self.assertEqual(err.read(), '') self.assertEqual(out.read(), 'Hello World.\n') self.assertEqual(result, 0) # test `pav log build test` # note: echo-ing hello world should not require anything to be built out.truncate(0) err.truncate(0) args = parser.parse_args(['build', str(test.id)]) log_cmd.run(self.pav_cfg, args) out.seek(0) err.seek(0) self.assertEqual(out.read(), '') # test `pav log kickoff test` # note: in general, kickoff.log should be an empty file out.truncate(0) err.truncate(0) args = parser.parse_args(['kickoff', str(test.id)]) result = log_cmd.run(self.pav_cfg, args) out.seek(0) err.seek(0) self.assertEqual(out.read(), '') self.assertEqual(err.read(), '') self.assertEqual(result, 0) log_cmd.outfile = sys.stdout log_cmd.outfile = sys.stderr
def test_rebuilds(self): """Make sure rebuilding works as expected.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args([ 'build', '-H', 'this', 'build_rebuild', '--rebuild', ]) build_cmd = commands.get_command(args.command_name) # type: RunCommand self.assertEqual(build_cmd.run(self.pav_cfg, args), 0) for test in build_cmd.last_tests: test.wait(timeout=3) # Make sure we actually built separate builds builds = [test.builder for test in build_cmd.last_tests] build_names = set([b.name for b in builds]) self.assertEqual(len(build_names), 4) result_matrix = { 'local1': [STATES.BUILD_DONE, STATES.BUILD_REUSED], 'local1a': [STATES.BUILD_REUSED, STATES.BUILD_DONE], 'nodes1': [STATES.BUILD_REUSED, STATES.BUILD_DONE], 'nodes1a': [STATES.BUILD_REUSED, STATES.BUILD_DONE], 'local2': [STATES.BUILD_DONE], 'nodes3': [STATES.BUILD_DONE], } orig_names = {} for test in build_cmd.last_tests: tname = test.name.split('.')[1] self.assertIn(test.status.current().state, result_matrix[tname], msg='Test {} status: {}'.format( test.name, test.status.current())) orig_names[test.name] = test.builder.name self.assertEqual(build_cmd.run(self.pav_cfg, args), 0) for test in build_cmd.last_tests: test.wait(timeout=3) # Make sure we actually built separate builds builds = [test.builder for test in build_cmd.last_tests] build_names = set([b.name for b in builds]) self.assertEqual(len(build_names), 4) for test in build_cmd.last_tests: expected_name = orig_names[test.name] + '-2' self.assertEqual(test._load_build_name(), expected_name, msg=test.name) origin = test.build_origin_path.resolve().name self.assertEqual(origin, expected_name, msg=test.name)
def test_cancel(self): """Test cancel command with no arguments.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this', 'cancel_test']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = run_cmd.errfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['cancel']) get_statuses(self.pav_cfg, args, StringIO()) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = cancel_cmd.errfile = StringIO() self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_no_sched(self): """Check that we get a reasonable error for a non-available scheduler.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', 'not_available']) run_cmd = commands.get_command(args.command_name) self.assertNotEqual(run_cmd.run(self.pav_cfg, args), 0)
def test_clean_with_invalid_date(self): """Test clean command with invalid arguments.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args( ['clean', '--older-than', '5 foo invalid']) clean_cmd = commands.get_command(args.command_name) clean_cmd.outfile = StringIO() clean_cmd.errfile = StringIO() self.assertEqual(clean_cmd.run(self.pav_cfg, args), errno.EINVAL)
def test_series_circle(self): """Test if it can detect circular references and that ordered: True works as intended.""" series_cmd = commands.get_command('series') arg_parser = arguments.get_parser() series_args = arg_parser.parse_args(['series', 'series_circle1']) self.assertRaises(pavilion.series_util.TestSeriesError, lambda: series_cmd.run(self.pav_cfg, series_args))
def test_cancel_status_flag(self): """Test cancel command with status flag.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args( ['run', '-H', 'this', 'cancel_test.test1']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['cancel', '-s']) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = StringIO() cancel_cmd.errfile = StringIO() self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_run(self): arg_parser = arguments.get_parser() args = arg_parser.parse_args( ['run', '-H', 'this', 'hello_world.world', 'hello_world.narf']) run_cmd = commands.get_command(args.command_name) self.assertEqual(run_cmd.run(self.pav_cfg, args), 0)
def test_wait_cancel(self): """Test cancel command after waiting for tests to start.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this', 'hello_world']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['cancel']) time.sleep(5) get_statuses(self.pav_cfg, args, StringIO()) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = StringIO() cancel_cmd.errfile = StringIO() self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_cancel_invalid_test(self): """Test cancel command with invalid test.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['cancel', '{}'.format(sys.maxsize)]) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = cancel_cmd.errfile = StringIO() self.assertEqual(cancel_cmd.run(self.pav_cfg, args), errno.EINVAL)
def test_cancel_cancelled_test(self): """Test cancelling a previously cancelled test.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', '-H', 'this', 'hello_world']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = StringIO() run_cmd.run(self.pav_cfg, args) args = arg_parser.parse_args(['cancel']) get_statuses(self.pav_cfg, args, StringIO()) cancel_cmd = commands.get_command(args.command_name) cancel_cmd.outfile = StringIO() cancel_cmd.errfile = StringIO() cancel_cmd.run(self.pav_cfg, args) self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_version_incompatibility(self): """Make sure incompatible versions exit gracefully when attempting to run.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', 'version_incompatible']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = io.StringIO() run_cmd.errfile = run_cmd.outfile self.assertEqual(run_cmd.run(self.pav_cfg, args), 22)
def test_run_status(self): """Tests run command with status flag.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args([ 'run', '-s', 'hello_world', ]) run_cmd = commands.get_command(args.command_name) self.assertEqual(run_cmd.run(self.pav_cfg, args), 0)