def test_clean_wait(self):
"""Test clean command after waiting for tests to finish."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args([
'run',
'-H', 'this',
'clean_test'
])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
time.sleep(5)
args = arg_parser.parse_args([
'clean'
])
clean_cmd = commands.get_command(args.command_name)
clean_cmd.outfile = StringIO()
clean_cmd.errfile = StringIO()
self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0)
def test_cancel_series_test(self):
"""Test cancel command with combination of series and tests."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(
['run', '-H', 'this', 'hello_world.hello', 'hello_world.world'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
tests = []
series_id = series.TestSeries.load_user_series_id(self.pav_cfg)
tests.append(series_id)
tests.extend(
series.TestSeries.from_id(self.pav_cfg, int(series_id[1:])).tests)
args = arg_parser.parse_args(
['cancel', tests[0],
str(tests[1]), str(tests[2])])
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = StringIO()
cancel_cmd.errfile = StringIO()
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_clean_with_older_than_flag(self):
"""Test clean command with multiple date formats."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['clean', '--older-than', '5 weeks'])
clean_cmd = commands.get_command(args.command_name)
clean_cmd.outfile = StringIO()
clean_cmd.errfile = StringIO()
self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0)
args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['clean', '--older-than', 'Jul 3 2019'])
clean_cmd = commands.get_command(args.command_name)
clean_cmd.outfile = StringIO()
clean_cmd.errfile = StringIO()
self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0)
def test_command_plugins(self):
"""Make sure command plugin loading is sane."""
# Get an empty pavilion config and set some config dirs on it.
pav_cfg = config.PavilionConfigLoader().load_empty()
# We're loading multiple directories of plugins - AT THE SAME TIME!
pav_cfg.config_dirs = [
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir'),
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir2')
]
plugins.initialize_plugins(pav_cfg)
commands.get_command('poof').run(pav_cfg, [])
commands.get_command('blarg').run(pav_cfg, [])
# Clean up our plugin initializations.
plugins._reset_plugins()
pav_cfg.config_dirs.append(
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir_conflicts'))
self.assertRaises(plugins.PluginError,
lambda: plugins.initialize_plugins(pav_cfg))
# Clean up our plugin initializations.
plugins._reset_plugins()
def test_cancel_series_test(self):
"""Test cancel command with combination of series and tests."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(
['run', '-H', 'this', 'cancel_test.test1', 'cancel_test.test2'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
tests = []
series_id = series.TestSeries.load_user_series_id(self.pav_cfg)
tests.append(series_id)
args = arg_parser.parse_args([
'cancel',
tests[0],
])
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = cancel_cmd.errfile = StringIO()
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_cancel_sched_check(self):
"""Cancel Test and make sure it is cancelled through scheduler."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this' 'hello_world2'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['cancel'])
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = StringIO()
cancel_cmd.errfile = StringIO()
test = []
series_id = series.TestSeries.load_user_series_id(self.pav_cfg)
test.append(series_id)
test_list = []
test_list.extend(
series.TestSeries.from_id(self.pav_cfg, int(test[0][1:])).tests)
for test_id in test_list:
test = PavTest.load(self.pav_cfg, test_id)
if test.status.current().state != STATES.COMPLETE:
sched = schedulers.get_scheduler_plugin(test.scheduler)
sched_status = sched.job_status(self.pav_cfg, test)
self.assertIn("SCHED_CANCELLED", str(sched_status))
def test_cancel_status_json(self):
"""Test cancel command with status flag and json flag."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(
['run', '-H', 'this', 'hello_world.world'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['cancel', '-s', '-j'])
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = StringIO()
cancel_cmd.errfile = StringIO()
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
results = cancel_cmd.outfile.getvalue().split('\n')[-1].strip().encode(
'UTF-8')
results = results[4:].decode('UTF-8')
results = json.loads(results)
self.assertNotEqual(len(results), 0)
def test_multi_build(self):
"""Make sure we can build multiple simultanious builds on
both the front-end and the nodes."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args([
'run',
'-H', 'this',
'build_parallel'
])
run_cmd = commands.get_command(args.command_name) # type: RunCommand
run_ret = run_cmd.run(self.pav_cfg, args)
run_cmd.outfile.seek(0)
self.assertEqual(run_ret, 0, msg=run_cmd.outfile.read())
for test in run_cmd.last_tests:
test.wait(timeout=4)
# Make sure we actually built separate builds
builds = [test.builder for test in run_cmd.last_tests]
build_names = set([b.name for b in builds])
self.assertEqual(len(build_names), 4)
for test in run_cmd.last_tests:
self.assertEqual(test.results['result'], 'PASS',
msg='Test {} status: {}'
.format(test.id, test.status.current()))
def test_clean(self):
"""Test clean command with no arguments."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test'])
run_cmd = commands.get_command(args.command_name)
run_cmd.silence()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['clean'])
clean_cmd = commands.get_command(args.command_name)
clean_cmd.silence()
self.assertEqual(clean_cmd.run(self.pav_cfg, args), 0)
def test_status_history(self):
# Testing that status works with history flag
status_cmd = commands.get_command('status')
out = io.StringIO()
status_cmd.outfile = out
parser = argparse.ArgumentParser()
status_cmd._setup_arguments(parser)
test = self._quick_test()
raw = schedulers.get_plugin('raw')
raw.schedule_test(self.pav_cfg, test)
end = time.time() + 5
while test.check_run_complete() is None and time.time() < end:
time.sleep(.1)
args = parser.parse_args(['--history', str(test.id)])
self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
out.seek(0)
output = out.readlines()[4:]
statuses = test.status.history()
self.assertEqual(len(output), len(statuses))
for i in range(len(output)):
self.assertTrue(statuses[i].state in output[i])
def test_local_builds_only(self):
"""Make sure we can just build multiple simultanious builds on
both the front-end and the nodes."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(
['build', '-H', 'this', '--local-builds-only', 'build_parallel'])
build_cmd = commands.get_command(args.command_name) # type: RunCommand
build_ret = build_cmd.run(self.pav_cfg, args)
build_cmd.outfile.seek(0)
self.assertEqual(build_ret, 0, msg=build_cmd.outfile.read())
for test in build_cmd.last_tests:
test.wait(timeout=10)
# Make sure we actually built separate builds
builds = [test.builder for test in build_cmd.last_tests]
build_names = set([b.name for b in builds])
self.assertEqual(len(build_names), 2)
for test in build_cmd.last_tests:
self.assertEqual(test.status.current().state,
STATES.BUILD_DONE,
msg='Test {} status: {}'.format(
test.id, test.status.current()))
def test_status_command_with_sched(self):
"""Test status command when test is 'SCHEDULED'."""
cfg = file_format.TestConfigLoader().validate({
'scheduler': 'raw',
'run': {
'env': {
'foo': 'bar',
},
'cmds': ['sleep 1'],
},
})
cfg['name'] = 'testytest'
test = self._quick_test(cfg, build=False, finalize=False)
test.build()
schedulers.get_plugin(test.scheduler) \
.schedule_test(self.pav_cfg, test)
status_cmd = commands.get_command('status')
status_cmd.outfile = io.StringIO()
parser = argparse.ArgumentParser()
status_cmd._setup_arguments(parser)
args = parser.parse_args([str(test.id)])
test.status.set(status_file.STATES.SCHEDULED, "faker")
self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
parser = argparse.ArgumentParser()
status_cmd._setup_arguments(parser)
args = parser.parse_args(['-j', str(test.id)])
test.status.set(status_file.STATES.SCHEDULED, "faker")
self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def test_build_parallel_lots(self):
"""Make sure building works beyond the parallel building limit."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args([
'run',
'-H', 'this',
'build_parallel_lots'
])
run_cmd = commands.get_command(args.command_name) # type: RunCommand
run_ret = run_cmd.run(self.pav_cfg, args)
run_cmd.outfile.seek(0)
self.assertEqual(run_ret, 0, msg=run_cmd.outfile.read())
for test in run_cmd.last_tests:
test.wait(timeout=5)
# Make sure we actually built separate builds
builds = [test.builder for test in run_cmd.last_tests]
build_names = set([b.name for b in builds])
self.assertEqual(len(build_names), 8)
for test in run_cmd.last_tests:
self.assertEqual(test.results['result'], 'PASS',
msg='Test {} status: {}'
.format(test.id, test.status.current()))
def test_status_summary(self):
# Testing that status works with summary flag
status_cmd = commands.get_command('status')
status_cmd.outfile = io.StringIO()
parser = argparse.ArgumentParser()
status_cmd._setup_arguments(parser)
arg_list = ['-s']
args = parser.parse_args(arg_list)
# Test that an empty working_dir fails correctly
self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
base_cfg = self._quick_test_cfg()
test_cfg1 = base_cfg.copy()
test_cfg1['name'] = 'test1'
test_cfg2 = base_cfg.copy()
test_cfg2['name'] = 'test2'
test_cfg3 = base_cfg.copy()
test_cfg3['name'] = 'test3'
configs = [test_cfg1, test_cfg2, test_cfg3]
tests = [self._quick_test(cfg) for cfg in configs]
for test in tests:
test.RUN_SILENT_TIMEOUT = 1
# Testing that summary flags return correctly
self.assertEqual(status_cmd.run(self.pav_cfg, args), 0)
def test_multi_build_fail(self):
"""Make sure we can build multiple simultanious builds on
both the front-end and the nodes."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args([
'run',
'-H', 'this',
'build_parallel_fail'
])
run_cmd = commands.get_command(args.command_name) # type: RunCommand
self.assertEqual(run_cmd.run(self.pav_cfg, args), 22)
# Make sure we actually built separate builds
builds = [test.builder for test in run_cmd.last_tests]
build_names = set([b.name for b in builds])
self.assertEqual(len(build_names), 4)
statuses = [test.status.current().state for test in run_cmd.last_tests]
statuses = set(statuses)
self.assertEqual(statuses, {STATES.ABORTED, STATES.BUILD_FAILED})
self.assertTrue(all([test.complete for test in run_cmd.last_tests]))
def test_run_timeouts(self):
"""Make sure run timeout file works as expected."""
run_cmd = commands.get_command('run')
run_cmd.silence()
arg_parser = arguments.get_parser()
# All test follow the same pattern seen above, but we can run them all
# at once, since a run timeout doesn't effect the others.
args = arg_parser.parse_args(['run', 'timeout_run_tests'])
self.assertEqual(run_cmd.run(self.pav_cfg, args), 0)
time.sleep(35)
correct_statuses = {
'timeout_run_tests.GoodRun': 'COMPLETE',
'timeout_run_tests.GoodRun2': 'COMPLETE',
'timeout_run_tests.GoodRun3': 'COMPLETE',
'timeout_run_tests.BadRun': 'RUN_TIMEOUT',
'timeout_run_tests.BadRun2': 'RUN_TIMEOUT',
'timeout_run_tests.BadRun3': 'RUN_TIMEOUT'
}
status_args = arg_parser.parse_args(['status'])
statuses = get_statuses(self.pav_cfg, status_args, io.StringIO())
for test_status in statuses:
self.assertEqual(correct_statuses[test_status['name']],
test_status['state'])
def test_sys_name_tracker(self):
"""Make sure the expected values are stored in the user.json file."""
user = utils.get_login()
sys_vars = system_variables.get_vars(True)
sys_name = sys_vars['sys_name']
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', 'hello_world'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = io.StringIO()
run_cmd.errfile = run_cmd.outfile
run_cmd.run(self.pav_cfg, args)
series = run_cmd.last_series
json_file = self.pav_cfg.working_dir / 'users'
json_file /= '{}.json'.format(user)
with json_file.open('r') as json_series_file:
data = json.load(json_series_file)
self.assertEqual(data[sys_name], series.sid)
def test_log_arguments(self):
log_cmd = commands.get_command('log')
parser = argparse.ArgumentParser()
log_cmd._setup_arguments(parser)
# run a simple test
test = self._quick_test(finalize=False)
raw = schedulers.get_plugin('raw')
raw.schedule_test(self.pav_cfg, test)
state = test.status.current().state
end = time.time() + 1
while ('ERROR' not in state and 'FAIL' not in state
and state != STATES.COMPLETE and time.time() < end):
time.sleep(.1)
# test `pav log run test`
args = parser.parse_args(['run', str(test.id)])
self.assertEqual(args.test, test.id)
out = io.StringIO()
err = io.StringIO()
log_cmd.outfile = out
log_cmd.errfile = err
result = log_cmd.run(self.pav_cfg, args)
err.seek(0)
out.seek(0)
self.assertEqual(err.read(), '')
self.assertEqual(out.read(), 'Hello World.\n')
self.assertEqual(result, 0)
# test `pav log build test`
# note: echo-ing hello world should not require anything to be built
out.truncate(0)
err.truncate(0)
args = parser.parse_args(['build', str(test.id)])
log_cmd.run(self.pav_cfg, args)
out.seek(0)
err.seek(0)
self.assertEqual(out.read(), '')
# test `pav log kickoff test`
# note: in general, kickoff.log should be an empty file
out.truncate(0)
err.truncate(0)
args = parser.parse_args(['kickoff', str(test.id)])
result = log_cmd.run(self.pav_cfg, args)
out.seek(0)
err.seek(0)
self.assertEqual(out.read(), '')
self.assertEqual(err.read(), '')
self.assertEqual(result, 0)
log_cmd.outfile = sys.stdout
log_cmd.outfile = sys.stderr
def test_rebuilds(self):
"""Make sure rebuilding works as expected."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args([
'build',
'-H',
'this',
'build_rebuild',
'--rebuild',
])
build_cmd = commands.get_command(args.command_name) # type: RunCommand
self.assertEqual(build_cmd.run(self.pav_cfg, args), 0)
for test in build_cmd.last_tests:
test.wait(timeout=3)
# Make sure we actually built separate builds
builds = [test.builder for test in build_cmd.last_tests]
build_names = set([b.name for b in builds])
self.assertEqual(len(build_names), 4)
result_matrix = {
'local1': [STATES.BUILD_DONE, STATES.BUILD_REUSED],
'local1a': [STATES.BUILD_REUSED, STATES.BUILD_DONE],
'nodes1': [STATES.BUILD_REUSED, STATES.BUILD_DONE],
'nodes1a': [STATES.BUILD_REUSED, STATES.BUILD_DONE],
'local2': [STATES.BUILD_DONE],
'nodes3': [STATES.BUILD_DONE],
}
orig_names = {}
for test in build_cmd.last_tests:
tname = test.name.split('.')[1]
self.assertIn(test.status.current().state,
result_matrix[tname],
msg='Test {} status: {}'.format(
test.name, test.status.current()))
orig_names[test.name] = test.builder.name
self.assertEqual(build_cmd.run(self.pav_cfg, args), 0)
for test in build_cmd.last_tests:
test.wait(timeout=3)
# Make sure we actually built separate builds
builds = [test.builder for test in build_cmd.last_tests]
build_names = set([b.name for b in builds])
self.assertEqual(len(build_names), 4)
for test in build_cmd.last_tests:
expected_name = orig_names[test.name] + '-2'
self.assertEqual(test._load_build_name(),
expected_name,
msg=test.name)
origin = test.build_origin_path.resolve().name
self.assertEqual(origin, expected_name, msg=test.name)
def test_cancel(self):
"""Test cancel command with no arguments."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this', 'cancel_test'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = run_cmd.errfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['cancel'])
get_statuses(self.pav_cfg, args, StringIO())
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = cancel_cmd.errfile = StringIO()
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_no_sched(self):
"""Check that we get a reasonable error for a non-available
scheduler."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', 'not_available'])
run_cmd = commands.get_command(args.command_name)
self.assertNotEqual(run_cmd.run(self.pav_cfg, args), 0)
def test_clean_with_invalid_date(self):
"""Test clean command with invalid arguments."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this', 'clean_test'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(
['clean', '--older-than', '5 foo invalid'])
clean_cmd = commands.get_command(args.command_name)
clean_cmd.outfile = StringIO()
clean_cmd.errfile = StringIO()
self.assertEqual(clean_cmd.run(self.pav_cfg, args), errno.EINVAL)
def test_series_circle(self):
"""Test if it can detect circular references and that ordered: True
works as intended."""
series_cmd = commands.get_command('series')
arg_parser = arguments.get_parser()
series_args = arg_parser.parse_args(['series', 'series_circle1'])
self.assertRaises(pavilion.series_util.TestSeriesError,
lambda: series_cmd.run(self.pav_cfg, series_args))
def test_cancel_status_flag(self):
"""Test cancel command with status flag."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(
['run', '-H', 'this', 'cancel_test.test1'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['cancel', '-s'])
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = StringIO()
cancel_cmd.errfile = StringIO()
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_run(self):
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(
['run', '-H', 'this', 'hello_world.world', 'hello_world.narf'])
run_cmd = commands.get_command(args.command_name)
self.assertEqual(run_cmd.run(self.pav_cfg, args), 0)
def test_wait_cancel(self):
"""Test cancel command after waiting for tests to start."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this', 'hello_world'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['cancel'])
time.sleep(5)
get_statuses(self.pav_cfg, args, StringIO())
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = StringIO()
cancel_cmd.errfile = StringIO()
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_cancel_invalid_test(self):
"""Test cancel command with invalid test."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['cancel', '{}'.format(sys.maxsize)])
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = cancel_cmd.errfile = StringIO()
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), errno.EINVAL)
def test_cancel_cancelled_test(self):
"""Test cancelling a previously cancelled test."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', '-H', 'this', 'hello_world'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = StringIO()
run_cmd.run(self.pav_cfg, args)
args = arg_parser.parse_args(['cancel'])
get_statuses(self.pav_cfg, args, StringIO())
cancel_cmd = commands.get_command(args.command_name)
cancel_cmd.outfile = StringIO()
cancel_cmd.errfile = StringIO()
cancel_cmd.run(self.pav_cfg, args)
self.assertEqual(cancel_cmd.run(self.pav_cfg, args), 0)
def test_version_incompatibility(self):
"""Make sure incompatible versions exit gracefully when attempting to
run."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', 'version_incompatible'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = io.StringIO()
run_cmd.errfile = run_cmd.outfile
self.assertEqual(run_cmd.run(self.pav_cfg, args), 22)
def test_run_status(self):
"""Tests run command with status flag."""
arg_parser = arguments.get_parser()
args = arg_parser.parse_args([
'run',
'-s',
'hello_world',
])
run_cmd = commands.get_command(args.command_name)
self.assertEqual(run_cmd.run(self.pav_cfg, args), 0)
