def test_sagemaker_context(microbenchmark_repeat, microbenchmark_range):
times_taken = []
for _ in range(microbenchmark_repeat):
timer_context = Timer()
with SagemakerSimulator():
with timer_context as t:
for _ in range(microbenchmark_range):
hook = get_smdebug_hook("keras")
times_taken.append(t.time_taken)
p95 = percentile(times_taken, 95)
assert p95 < 12 # current mean = ~12 seconds
def test_get_smdebug_hook_use_smdebug(use_smdebug, microbenchmark_repeat,
microbenchmark_range, monkeypatch):
monkeypatch.setenv("USE_SMDEBUG", use_smdebug)
times_taken = []
for _ in range(microbenchmark_repeat):
timer_context = Timer()
with timer_context as t:
for _ in range(microbenchmark_range):
hook = get_smdebug_hook("keras")
times_taken.append(t.time_taken)
p95 = percentile(times_taken, 95)
# mean time taken with use_smdebug == 0 is 3 seconds
# mean time taken with use_smdebug == 1 is 21 seconds
threshold = 3 if use_smdebug == "0" else 21
assert p95 < threshold
def wrapped(*args):
horizons.globals.db = db
if not manual_session and not use_fixture:
s, p = new_session(mapgen=mapgen,
human_player=human_player,
ai_players=ai_players)
elif use_fixture:
path = os.path.join(TEST_FIXTURES_DIR, use_fixture + '.sqlite')
if not os.path.exists(path):
raise Exception('Savegame %s not found' % path)
s = load_session(path)
timelimit = Timer(handler)
timelimit.start(timeout)
try:
if use_fixture:
return func(s, *args)
elif not manual_session:
return func(s, p, *args)
else:
return func(*args)
finally:
try:
if use_fixture:
s.end(remove_savegame=False, keep_map=True)
elif not manual_session:
s.end()
except:
pass
# An error happened after cleanup after an error.
# This is ok since cleanup is only defined to work when invariants are in place,
# but the first error could have violated one.
# Therefore only use failsafe cleanup:
finally:
SPTestSession.cleanup()
timelimit.stop()
def wrapped():
test_name = '{}.{}'.format(func.__module__, func.__name__)
# when running under coverage, enable it for subprocesses too
if os.environ.get('RUNCOV'):
executable = ['coverage', 'run']
else:
executable = [sys.executable]
args = executable + [
'run_uh.py', '--sp-seed',
str(RANDOM_SEED), '--gui-test', test_name
]
if use_fixture:
path = os.path.join(TEST_FIXTURES_DIR, use_fixture + '.sqlite')
if not os.path.exists(path):
raise Exception('Savegame {} not found'.format(path))
args.extend(['--load-game', path])
elif use_dev_map:
args.append('--start-dev-map')
elif use_scenario:
args.extend(['--start-scenario', use_scenario + '.yaml'])
if ai_players:
args.extend(['--ai-players', str(ai_players)])
if additional_cmdline:
args.extend(additional_cmdline)
try:
# if nose does not capture stdout, then most likely someone wants to
# use a debugger (he passed -s at the cmdline). In that case, we will
# redirect stdout/stderr from the gui-test process to the testrunner
# process.
# nose alters sys.stdout so that fileno is no longer its property
sys.stdout.fileno()
stdout = sys.stdout
stderr = sys.stderr
nose_captured = False
except io.UnsupportedOperation:
# if nose captures stdout, we can't redirect to sys.stdout, as that was
# replaced by StringIO. Instead we capture it and return the data at the
# end.
stdout = subprocess.PIPE
stderr = subprocess.PIPE
nose_captured = True
# Activate fail-fast, this way the game will stop running when for example the
# savegame could not be loaded (instead of showing an error popup)
env = os.environ.copy()
env['FAIL_FAST'] = '1'
env['UH_USER_DIR'] = _user_dir or TEST_USER_DIR
env['PYTHONFAULTHANDLER'] = '1'
# Start game
proc = subprocess.Popen(args,
stdout=stdout,
stderr=stderr,
env=env)
def handler(signum, frame):
proc.kill()
raise TestFailed(
'\n\nTest run exceeded {:d}s time limit'.format(timeout))
timelimit = Timer(handler)
timelimit.start(timeout)
stdout, stderr = proc.communicate()
if cleanup_userdir:
recreate_userdir()
if proc.returncode != 0:
if nose_captured:
if stdout:
print(stdout)
if b'Traceback' not in stderr:
stderr += b'\nNo usable error output received, possibly a segfault.'
raise TestFailed('\n\n' + stderr.decode('ascii', 'ignore'))
else:
raise TestFailed()
def pytest_pyfunc_call(pyfuncitem):
"""
Tests marked with gui_test will, instead of executing the test function, start a new
process with the game and run the test function code inside that process.
"""
info = pyfuncitem.get_marker('gui_test')
if not info:
return
tmpdir = pyfuncitem._request.getfixturevalue('tmpdir')
use_fixture = info.kwargs.get('use_fixture', None)
use_dev_map = info.kwargs.get('use_dev_map', False)
use_scenario = info.kwargs.get('use_scenario', None)
ai_players = info.kwargs.get('ai_players', 0)
additional_cmdline = info.kwargs.get('additional_cmdline', None)
timeout = info.kwargs.get('timeout', 15 * 60)
modify_user_dir = info.kwargs.get('_modify_user_dir', lambda v: v)
test_name = '{}.{}'.format(pyfuncitem.module.__name__, pyfuncitem.name)
# when running under coverage, enable it for subprocesses too
if os.environ.get('RUNCOV'):
executable = ['coverage', 'run']
else:
executable = [sys.executable]
args = executable + [
'run_uh.py', '--sp-seed',
str(RANDOM_SEED), '--gui-test', test_name
]
if use_fixture:
path = os.path.join(TEST_FIXTURES_DIR, use_fixture + '.sqlite')
if not os.path.exists(path):
raise Exception('Savegame {} not found'.format(path))
args.extend(['--load-game', path])
elif use_dev_map:
args.append('--start-dev-map')
elif use_scenario:
args.extend(['--start-scenario', use_scenario + '.yaml'])
if ai_players:
args.extend(['--ai-players', str(ai_players)])
if additional_cmdline:
args.extend(additional_cmdline)
user_dir = modify_user_dir(tmpdir.join('user_dir'))
env = os.environ.copy()
# Setup temporary user directory for each test
env['UH_USER_DIR'] = str(user_dir)
# Activate fail-fast, this way the game will stop running when for example the savegame
# could not be loaded (instead of showing an error popup)
env['FAIL_FAST'] = '1'
# Show additional information (e.g. about threads) when the interpreter crashes
env['PYTHONFAULTHANDLER'] = '1'
if pyfuncitem.config.option.capture == 'no':
# if pytest does not capture stdout, then most likely someone wants to
# use a debugger (he passed -s at the cmdline). In that case, we will
# redirect stdout/stderr from the gui-test process to the testrunner
# process.
stdout = sys.stdout
stderr = sys.stderr
output_captured = False
else:
# if pytest captures stdout, we can't redirect to sys.stdout, as that
# was replaced by a custom object. Instead we capture it and return the
# data at the end.
stdout = subprocess.PIPE
stderr = subprocess.PIPE
output_captured = True
# Start game
proc = subprocess.Popen(args, stdout=stdout, stderr=stderr, env=env)
def handler(signum, frame):
proc.kill()
raise TestFailed(
'\n\nTest run exceeded {:d}s time limit'.format(timeout))
timelimit = Timer(handler)
timelimit.start(timeout)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
if output_captured:
if stdout:
print(stdout)
if b'Traceback' not in stderr:
stderr += b'\nNo usable error output received, possibly a segfault.'
raise TestFailed(stderr.decode('ascii'))
return True
