def run_tests():
"""
Run some basic testfiles for the simulator backend in /topo/custom/tests folder;
Note that this code is partially obsolete, dig into the code for details. Also
note that test_routing_with_overutil.py and test_changepath_stacked.py are supposed
to fail at the moment (represent corner cases that are çurrently not properly supported).
"""
success = []
failure = []
# get all test files
testfiles = []
blacklist = ["__init__.py", "testutil.py"]
testdir = os.path.join(os.getcwd(), "topo", "custom", "tests")
for root, dirs, files in os.walk(testdir):
for file in files:
if file in blacklist: continue
if file.endswith(".py"):
testfiles.append(file)
logger.info("tests: %s" % ",".join(testfiles))
# run them one by one and print a summary afterwards
for testname in testfiles:
# please not that print is used because it logs to stdout
print("+------------------------+")
print("| %s" % testname)
print("+------------------------+")
ctx = Context()
topo = Topology(ctx, filename='custom/tests/%s' % testname)
try:
sim = Simulator(ctx)
errors = sim.run()
except Exception as e:
failure.append((testname, ['test crashed: %s' % str(e)]))
continue
if errors == None:
# if the simulator didn't return an array the
# on_test_finished method is not implemented properly
failure.append(
(testname,
['test did not return an array, prob. not implemented']))
continue
if len(errors) == 0:
success.append(testname)
else:
failure.append((testname, errors))
logger.info("+------ Test Results ----+")
for test in success:
logger.info("| %30s PASSED" % test)
if len(failure) > 0: logger.info("|--->")
for test, errors in failure:
logger.info("| %30s FAILED" % test)
logger.info("+------------------------+")
if len(failure) == 0:
logger.info("| ALL TESTS PASSED")
logger.info("+------------------------+")
def setUp(self):
self.history = 4
with open('./data/replay_buffer.pkl', 'rb') as file:
self.replay_buffer = pickle.load(file)
env_to_use = 'Pendulum-v0'
env = gym.make(env_to_use)
self.history = 4
self.simulator = Simulator(self.history, env)
class MatcherTestCase(unittest.TestCase):
"""Tests the various Matcher subclasses"""
all_matchers = [GeogMatcher]
def setUp(self):
self.sim = Simulator()
self.sim.simulate(10, 13579)
def test_matcher(self):
"""Test various matcher subclasses"""
def test_empty_matcher(MatcherSub):
"""Test empty Matcher returns valid empty results"""
## Test that an empty set returns valid, emtpy results
matcher_sub = MatcherSub(set([]))
unmatched = matcher_sub.get_unmatched()
best_matches = matcher_sub.get_best_matches()
self.assertEqual(unmatched, set([]));
self.assertEqual(best_matches, set([]));
def test_matcher_matches(MatcherSub, expected_matches, expected_unmatches):
"""Spin up Matcher subclass and confirm expected results"""
users = self.sim.get_users()
self.assertEqual(len(users),10, "Sanity check simulator user count")
matcher_sub = MatcherSub(users,seed=1234)
best_matches = set([(tup[0].user_id, tup[1].user_id) for tup in matcher_sub.get_best_matches() ])
unmatched = set([user.user_id for user in matcher_sub.get_unmatched()])
print best_matches
print unmatched
for match in expected_matches:
self.assertIn(match, best_matches, "Expected match not found - perhaps the algo or simulator changed?")
for unmatch in expected_unmatches:
self.assertIn(unmatch, unmatched, "Expected unmatched not found - perhaps the algo or simulator changed?")
## Test empty matcher return valid results
for MatcherSub in self.all_matchers:
test_empty_matcher(MatcherSub)
## For each Matcher subclass, work out what the answers should be
## given the simulator's users, and record the answers here
geog_matched = [(7736498, 2837615), (5072324, 795755), (4454245, 3882723)]
geog_unmatched = [1510961, 8148259, 6505708, 2432669]
## Feed the expected results into 3-tuples below to be tested
## (MatcherSubclass , expected_match_results, expected_unmatched_results)
test_cases = [
(GeogMatcher, geog_matched,geog_unmatched)
]
## Test full matchers return correct results
for match_case in test_cases:
test_matcher_matches(*match_case)
def setUp(self):
"""
Test Preparation
The simulator should deliver the time change signal in intervals of
one second divided by TEST_SPEED.
Since the simulator delivers event through the QtCore event loop, we
need to setup a qt application.
"""
self.app = QtCore.QCoreApplication([])
self.history = []
self.time_step = 1 / TEST_SPEED
self.simulator = Simulator(self._callback)
# state variables
self.simulation_time = None
self.t_elapsed = 0
def get_users(self):
"""Read in the survey data or simulate new data"""
self.users = self.get_users_parser(APIParser, self.survey_api_survey_id)
print self.users
if self.users is None:
if os.path.isfile(self.survey_data_filename):
self.users = self.get_users_parser(TextParser, self.survey_data_filename)
else:
print "Simulating data"
## Simulate some number of survey responses
sim = Simulator()
sim.simulate(200) ## By adding a seed parameter, you can have reproducible sims
self.users = sim.get_users()
# ## Print some summaries
# for user in users:
# user.to_string()
# Print some descriptive stats
print "# Users", len(self.users), [u.name for u in self.users]
def setUp(self):
self.sim = Simulator()
self.sim.simulate(10, 13579)
def main():
# -w (run preprocessor write)
if args.run_preprocessor:
run_preprocessor()
exit(0)
# -r (run preprocessor read)
if args.run_preprocessor_read:
run_preprocessor_read()
exit(0)
# -a (run aggregated plotter)
if args.aggregated_plots:
if not args.plotters:
logger.error("The -u option is required for -a. Exit now.")
exit(1)
if args.plotters == 'util/agg_preprocessor.py':
args.run_preprocessor = args.aggregated_plots
run_preprocessor()
exit(0)
run_aggregated_plotter()
exit(0)
# -p and -u (run plotter from pickle file)
if args.picklefile:
run_plotter()
exit(0)
# -t (run tests)
if args.run_tests:
run_tests()
exit(0)
# make sure the file that should be run exists
if not args.filename:
logger.error('The -f options is required. Exit now.')
exit(1)
try:
# TODO: currently, filenames are relative to the topo folder
topo_folder = os.path.join(os.getcwd(), 'topo')
filename = args.filename.strip()
if not filename.endswith('.py'): filename = filename + '.py'
# make sure the file that should be run exists
if not os.path.exists(os.path.join(topo_folder, filename)):
scripts = get_scripts(topo_folder)
print(scripts)
for script in scripts:
if filename == os.path.basename(script):
logger.info('use file: %s' % script)
filename = script
break
else:
logger.error('The file specified via -f was not found: %s' %
str(filename))
# fallback to default
topo_folder = os.path.join(os.getcwd(), 'topo')
filename = os.path.join(topo_folder, 'custom/pbce/exp800-2.py')
logger.error('Using fallback: %s' % filename)
if not os.path.exists(os.path.join(topo_folder, filename)):
logger.error('Fallback failed')
exit(1)
# create a new experiment context that stores all the data; this is
# the central object that is basically injected everywhere from now on; try not
# to add anything here except raw objects (i.e., no functions) due to
# serialization issues
ctx = Context()
ctx.started = time.time()
timer = Timer(ctx, 'main')
timer.start('total_time')
# enforce specific seed (stored in ctx)
if args.seed:
logger.info('Global seed set to %d via CLI' % int(args.seed))
ctx.seed = int(args.seed)
# only run the scenario generator (neither the simulator nor the DTS/RSA algorithms
# are executed)
if args.run_scenario_generator:
logger.info(
'Note: -g flag is set, i.e., the scenario generator ctx flag is active and the simulator will not be executed'
)
ctx.run_scenario_generator = True
# inject config from configfile into ctx
# this is useful if the tool is used in an automated fashion
if args.config:
with open(args.config, 'r') as file:
config = json.loads(file.read())
ctx.configfile = args.config
ctx.config = config
#logger.info(str(config) + " " + args.filename)
# create topology
topo = Topology(ctx, filename=filename)
if args.run_scenario_generator:
logger.info(
'Run scenario generator instead of simulator (-g flag was set)'
)
ctx.statistics['sim.skipped'] = 1
# finally run the simulator if requested
if not ctx.skip_main_simulation == True:
sim = Simulator(ctx)
sim.run()
# print the statistics
print_statistics(ctx)
# save the aggregated statistics (statistics.json)
if ctx.configfile:
statistics_file = os.path.join(os.path.dirname(ctx.configfile),
'statistics.json')
with open(statistics_file, 'w') as file:
file.write(json.dumps(ctx.statistics))
# saves a pickle result file from ctx to access all raw data later on
# (skipped if -n flag is used)
if not args.nopickle:
if ctx.scenario is not None:
timer.stop()
return
#raise Exception("ctx.scenario is set -> not possible to create a pickle file (-n option is mandatory here!")
if ctx.configfile:
sys.setrecursionlimit(100000)
result_file = os.path.join(os.path.dirname(ctx.configfile),
'result.pickle')
pickle.dump(ctx,
open(result_file, "wb"),
protocol=pickle.HIGHEST_PROTOCOL)
# deprecated
if args.plotters:
logger.info("run plotters..")
plotters = args.plotters.split(',')
# handle plotters
for plotter in plotters:
modulepath = "plotter.%s" % plotter.replace('.py', '').replace(
os.sep, '.')
modulepath = modulepath.replace('..', '.')
logger.info("loading plotter: %s" % modulepath)
importlib.import_module(modulepath).plot(ctx)
timer.stop()
except TimeoutError as e:
timer.stop()
# maximum time has exceeded
logger.info("Timelimit exceeded, abort")
# create statistics file (this is not technically an error, some experiments are just
# running too long)
if ctx.configfile:
statistics_file = os.path.join(os.path.dirname(ctx.configfile),
'statistics.json')
ctx.statistics['hit_timelimit'] = time.time() - ctx.started
print_statistics(ctx)
with open(statistics_file, 'w') as file:
file.write(json.dumps(ctx.statistics))
# still create an error message for quick checks
exc_string = traceback.format_exc()
if ctx.configfile:
error_file = os.path.join(os.path.dirname(ctx.configfile),
'timeout-error.txt')
with open(error_file, 'w') as file:
file.write(exc_string)
# finally print the exception and exit
print("Exception:")
print('-' * 60)
print(exc_string)
print('-' * 60)
exit(0)
except Exception as e:
timer.stop()
print("Exception:")
print('-' * 60)
exc_string = traceback.format_exc()
print(exc_string)
print('-' * 60)
# save the aggregated statistics (statistics.json)
if ctx.configfile:
error_file = os.path.join(os.path.dirname(ctx.configfile),
'error.txt')
with open(error_file, 'w') as file:
file.write(exc_string)
raise e
class MatcherTestCase(unittest.TestCase):
"""Tests the various Matcher subclasses"""
all_matchers = [GeogMatcher]
def setUp(self):
self.sim = Simulator()
self.sim.simulate(10, 13579)
def test_matcher(self):
"""Test various matcher subclasses"""
def test_empty_matcher(MatcherSub):
"""Test empty Matcher returns valid empty results"""
## Test that an empty set returns valid, emtpy results
matcher_sub = MatcherSub(set([]))
unmatched = matcher_sub.get_unmatched()
best_matches = matcher_sub.get_best_matches()
self.assertEqual(unmatched, set([]))
self.assertEqual(best_matches, set([]))
def test_matcher_matches(MatcherSub, expected_matches,
expected_unmatches):
"""Spin up Matcher subclass and confirm expected results"""
users = self.sim.get_users()
self.assertEqual(len(users), 10,
"Sanity check simulator user count")
matcher_sub = MatcherSub(users, seed=1234)
best_matches = set([(tup[0].user_id, tup[1].user_id)
for tup in matcher_sub.get_best_matches()])
unmatched = set(
[user.user_id for user in matcher_sub.get_unmatched()])
print best_matches
print unmatched
for match in expected_matches:
self.assertIn(
match, best_matches,
"Expected match not found - perhaps the algo or simulator changed?"
)
for unmatch in expected_unmatches:
self.assertIn(
unmatch, unmatched,
"Expected unmatched not found - perhaps the algo or simulator changed?"
)
## Test empty matcher return valid results
for MatcherSub in self.all_matchers:
test_empty_matcher(MatcherSub)
## For each Matcher subclass, work out what the answers should be
## given the simulator's users, and record the answers here
geog_matched = [(7736498, 2837615), (5072324, 795755),
(4454245, 3882723)]
geog_unmatched = [1510961, 8148259, 6505708, 2432669]
## Feed the expected results into 3-tuples below to be tested
## (MatcherSubclass , expected_match_results, expected_unmatched_results)
test_cases = [(GeogMatcher, geog_matched, geog_unmatched)]
## Test full matchers return correct results
for match_case in test_cases:
test_matcher_matches(*match_case)
def setUp(self):
self.sim = Simulator()
self.sim.simulate(10, 13579)
def test_blank_simulator(self):
blankSim = Simulator()
self.assertEqual(blankSim.get_users(),set([]))
self.assertEqual(blankSim.CAREER_STAGES, {"Student": 10, "Entry": 15, "Early":25, "Middle": 40, "Late": 10})
def setUp(self):
self.sim = Simulator()
class SimulatorTestCase(unittest.TestCase):
def setUp(self):
self.sim = Simulator()
def tearDown(self):
self.sim = None
def test_simulator(self):
self.assertEqual(self.sim.get_users(),set([]))
self.assertEqual(self.sim.CAREER_STAGES, {"Student": 10, "Entry": 15, "Early":25, "Middle": 40, "Late": 10})
def test_simulate(self):
"""Tests the simulate and clear_users methods
Since multiple calls to simulate continue to grow the group,
the test demonstrates that the group grows. It also tests that
the users set can be cleared.
"""
## Simulate the first 5 users
self.sim.simulate(5,1234)
users = list(sorted(
self.sim.get_users(),
cmp=lambda x,y : cmp(x.user_id,y.user_id)
))
print users
expected_results = {
4 : (8698081,"Melinda Thompson"),
3 : (8453420, "Freddie Bahlmann"),
2 : (8198783, "Gary Paddock"),
1 : (3966593, "Kevin Wilson"),
0 : (67423, "Scott Lopez")
}
for i in range(5):
self.assertEqual((users[i].user_id,users[i].name),expected_results[i])
## Clear the simulated users, and resimulate the same users
self.sim.clear_users()
self.sim.simulate(5,1234)
users = list(sorted(
self.sim.get_users(),
cmp=lambda x,y : cmp(x.user_id,y.user_id)
))
print users
expected_results = {
4 : (8698081,"Melinda Thompson"),
3 : (8453420, "Freddie Bahlmann"),
2 : (8198783, "Gary Paddock"),
1 : (3966593, "Kevin Wilson"),
0 : (67423, "Scott Lopez")
}
for i in range(5):
self.assertEqual((users[i].user_id,users[i].name),expected_results[i])
## Add 5 more users on top of existing users.
## Since the same seed is used, it should be tempted
## to duplicate the previous users, but the simulate
## method should prevent this from happening
self.sim.simulate(5,1234)
users = list(sorted(
self.sim.get_users(),
cmp=lambda x,y : cmp(x.user_id,y.user_id)
))
print users
expected_results = {
9 : (8698081,"Melinda Thompson"),
8 : (8698080, "Angela Jones"),
7 : (8453420, "Freddie Bahlmann"),
6 : (8453419, "Marilynn Orr"),
5 : (8198783, "Gary Paddock"),
4 : (8198781, "Tiffany Mchaney"),
3 : (3966593, "Kevin Wilson"),
2 : (3966592, "Donna Yu"),
1 : (67424, "Patrice Newell"),
0 : (67423, "Scott Lopez")
}
for i in range(5):
self.assertEqual((users[i].user_id,users[i].name),expected_results[i])
def test_blank_simulator(self):
blankSim = Simulator()
self.assertEqual(blankSim.get_users(),set([]))
self.assertEqual(blankSim.CAREER_STAGES, {"Student": 10, "Entry": 15, "Early":25, "Middle": 40, "Late": 10})
class MyTestCase(unittest.TestCase):
"""
hello
"""
def setUp(self):
self.history = 4
with open('./data/replay_buffer.pkl', 'rb') as file:
self.replay_buffer = pickle.load(file)
env_to_use = 'Pendulum-v0'
env = gym.make(env_to_use)
self.history = 4
self.simulator = Simulator(self.history, env)
def test_format_buffer(self):
# Separate into prev_observations
# previous actions
# next observations
# and reward for next observation
p_obs, p_a, n_o, p_r = self.simulator._format_buffer(
self.replay_buffer)
# Check prev Observations
first_obs = np.array([
self.replay_buffer[i][0] for i in range(self.history - 1, -1, -1)
]).reshape(-1)
self.assert_(np.array_equal(first_obs, p_obs[0]))
pos = 200
second_obs = np.array([
self.replay_buffer[i][0]
for i in range(pos + self.history - 1, pos - 1, -1)
]).reshape(-1)
self.assert_(np.array_equal(second_obs, p_obs[197]))
pos = 400
third_obs = np.array([
self.replay_buffer[i][0]
for i in range(pos + self.history - 1, pos - 1, -1)
]).reshape(-1)
self.assert_(np.array_equal(third_obs, p_obs[394]))
# Check prev Actions
first_act = np.array([
self.replay_buffer[i][1] for i in range(self.history - 1, -1, -1)
]).reshape(-1)
self.assert_(np.array_equal(first_act, p_a[0]))
pos = 200
second_act = np.array([
self.replay_buffer[i][1]
for i in range(pos + self.history - 1, pos - 1, -1)
]).reshape(-1)
self.assert_(np.array_equal(second_act, p_a[197]))
pos = 400
third_act = np.array([
self.replay_buffer[i][1]
for i in range(pos + self.history - 1, pos - 1, -1)
]).reshape(-1)
self.assert_(np.array_equal(third_act, p_a[394]))
# Check next observation
first_no = self.replay_buffer[self.history - 1][2]
print(first_no.shape, n_o[0].shape)
self.assert_(np.array_equal(first_no, n_o[0]))
pos = 200
second_no = self.replay_buffer[pos + self.history - 1][2]
self.assert_(np.array_equal(second_no, n_o[197]))
pos = 400
third_no = self.replay_buffer[pos + self.history - 1][2]
self.assert_(np.array_equal(third_no, n_o[394]))
# Check reward
first_r = self.replay_buffer[self.history - 1][3]
print(first_r.shape, n_o[0].shape)
self.assert_(np.array_equal(first_r, p_r[0]))
pos = 200
second_r = self.replay_buffer[pos + self.history - 1][3]
self.assert_(np.array_equal(second_r, p_r[197]))
pos = 400
third_r = self.replay_buffer[pos + self.history - 1][3]
self.assert_(np.array_equal(third_r, p_r[394]))
def test_train(self):
pass
class BasicOperation(unittest.TestCase):
""" Basic operation of the simulator """
def setUp(self):
"""
Test Preparation
The simulator should deliver the time change signal in intervals of
one second divided by TEST_SPEED.
Since the simulator delivers event through the QtCore event loop, we
need to setup a qt application.
"""
self.app = QtCore.QCoreApplication([])
self.history = []
self.time_step = 1 / TEST_SPEED
self.simulator = Simulator(self._callback)
# state variables
self.simulation_time = None
self.t_elapsed = 0
def _callback(self, t):
self.simulation_time = t
def _create_time_range(self, seconds):
start = datetime(2013, 12, 4, 9)
end = start + timedelta(seconds=seconds)
return start, end
def _step_time(self):
""" a helper function that increases the time step """
sleep(self.time_step)
self.t_elapsed += self.time_step
def test_delivery(self):
""" Tests signal delivery by the simulator """
# Simulate a time range of 2 seconds
duration = 2
# Allow a bit of leeway for the expected simulation end time
max_t = (duration + 1.5) / TEST_SPEED
min_t = (duration - 0.2) / TEST_SPEED
t_range = self._create_time_range(duration)
self.simulator.configure(t_range, TEST_SPEED)
self.simulator.start()
while (self.t_elapsed < max_t
and (self.simulation_time is None
or self.simulation_time < t_range[1])):
self.app.processEvents()
self._step_time()
self.assertLess(
self.t_elapsed, max_t,
'Events were not delivered in time (' + str(self.t_elapsed) + ')')
self.assertGreater(self.t_elapsed, min_t, 'Event delivery too fast')
def test_start_with_external_signal(self):
signal_emitter = SignalEmitter()
t_range = self._create_time_range(3600)
start_time = t_range[0]
dt = timedelta(seconds=1800)
self.simulator.configure(t_range, 0, signal_emitter.test_signal, dt)
self.simulator.start()
self.app.processEvents()
self.assertEqual(self.simulation_time, start_time + dt,
'First step was not executed immediately')
signal_emitter.test_signal.emit()
self.app.processEvents()
self.assertEqual(self.simulation_time, start_time + 2 * dt,
'Simulator did not step on external signal')
# The simulation should now be finished
signal_emitter.test_signal.emit()
self.app.processEvents()
self.assertEqual(self.simulation_time, start_time + 2 * dt,
'Simulator has not ended as expected')
def test_pause(self):
""" Tests pausing the simulator """
duration = 3
max_t = 8 / TEST_SPEED
min_t = 5 / TEST_SPEED
t_range = self._create_time_range(duration)
self.simulator.configure(t_range, TEST_SPEED)
self.simulator.start()
# No events should be delivered during the pause
while (self.t_elapsed < max_t
and (self.simulation_time is None
or self.simulation_time < t_range[1])):
self.app.processEvents()
if self.t_elapsed == 2 / TEST_SPEED:
self.simulator.pause()
elif self.t_elapsed == 5 / TEST_SPEED:
self.simulator.start()
self._step_time()
self.assertLess(
self.t_elapsed, max_t,
'Events were not delivered in time (' + str(self.t_elapsed) + ')')
self.assertGreater(self.t_elapsed, min_t, 'Event delivery too fast')
def test_stop(self):
""" Tests premature simulation stop """
duration = 3
max_t = 5 / TEST_SPEED
t_range = self._create_time_range(duration)
self.simulator.configure(t_range, TEST_SPEED)
self.simulator.start()
while self.t_elapsed < max_t:
self.app.processEvents()
if self.t_elapsed == 2 / TEST_SPEED:
self.simulator.stop()
self._step_time()
self.assertLess(self.simulation_time, t_range[1])
