def test_shutdown(self, monotonic_mock):
checkerscript_path = '/dev/null'
monotonic_mock.return_value = 10
master_loop = MasterLoop(self.connection, self.state_db_conn,
'service1', checkerscript_path, None, 2, 1,
10, '0.0.%s.1', b'secret', {}, DummyQueue())
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET start=NOW()')
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=0')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 0)')
master_loop.shutting_down = True
master_loop.supervisor.queue_timeout = 0.01
monotonic_mock.return_value = 20
# Will return False because no messages yet
self.assertFalse(master_loop.step())
with transaction_cursor(self.connection) as cursor:
cursor.execute(
'SELECT COUNT(*) FROM scoring_flag WHERE placement_start IS NOT NULL'
)
self.assertEqual(cursor.fetchone()[0], 0)
cursor.execute('SELECT COUNT(*) FROM scoring_statuscheck')
self.assertEqual(cursor.fetchone()[0], 0)
def test_down(self, monotonic_mock):
checkerscript_path = os.path.join(os.path.dirname(__file__),
'integration_down_checkerscript.py')
monotonic_mock.return_value = 10
master_loop = MasterLoop(self.connection, self.state_db_conn,
'service1', checkerscript_path, None, 2, 1,
10, '0.0.%s.1', b'secret', {}, DummyQueue())
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET start=NOW()')
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=0')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 0)')
monotonic_mock.return_value = 20
master_loop.supervisor.queue_timeout = 0.01
# Checker Script gets started, will return False because no messages yet
self.assertFalse(master_loop.step())
master_loop.supervisor.queue_timeout = 10
while master_loop.step():
pass
with transaction_cursor(self.connection) as cursor:
cursor.execute(
'SELECT COUNT(*) FROM scoring_flag WHERE placement_end IS NOT NULL'
)
self.assertEqual(cursor.fetchone()[0], 1)
cursor.execute('SELECT COUNT(*) FROM scoring_statuscheck')
self.assertEqual(cursor.fetchone()[0], 1)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=2 AND tick=0')
self.assertEqual(cursor.fetchone()[0], CheckResult.DOWN.value)
def setUp(self):
self.master_loop = MasterLoop(self.connection, 'service1', '/dev/null',
None, 2, 8, 10, '0.0.%s.1', b'secret',
{}, DummyQueue())
class MasterTest(DatabaseTestCase):
fixtures = ['tests/checker/fixtures/master.json']
def setUp(self):
self.master_loop = MasterLoop(self.connection, 'service1', '/dev/null',
None, 2, 8, 10, '0.0.%s.1', b'secret',
{}, DummyQueue())
def test_handle_flag_request(self):
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET start=NOW()')
task_info = {
'service': 'service1',
'_team_id': 2,
'team': 92,
'tick': 1
}
params1 = {'tick': 1}
resp1 = self.master_loop.handle_flag_request(task_info, params1)
params2 = {'tick': 1}
resp2 = self.master_loop.handle_flag_request(task_info, params2)
params3 = {'tick': 1, 'payload': 'TmV2ZXIgZ28='}
resp3 = self.master_loop.handle_flag_request(task_info, params3)
self.assertEqual(resp1, resp2)
self.assertNotEqual(resp1, resp3)
params4 = {'tick': 2}
resp4 = self.master_loop.handle_flag_request(task_info, params4)
params5 = {'tick': 2}
resp5 = self.master_loop.handle_flag_request(task_info, params5)
self.assertEqual(resp4, resp5)
self.assertNotEqual(resp1, resp4)
params6 = {}
self.assertIsNone(
self.master_loop.handle_flag_request(task_info, params6))
# Changing the start time changes all flags
with transaction_cursor(self.connection) as cursor:
# SQLite syntax for tests
cursor.execute(
'UPDATE scoring_gamecontrol SET start=DATETIME("now", "+1 hour")'
)
resp1_again = self.master_loop.handle_flag_request(task_info, params1)
resp4_again = self.master_loop.handle_flag_request(task_info, params4)
self.assertNotEqual(resp1, resp1_again)
self.assertNotEqual(resp4, resp4_again)
def test_handle_result_request(self):
task_info = {
'service': 'service1',
'_team_id': 2,
'team': 92,
'tick': 1
}
param = CheckResult.OK.value
start_time = datetime.datetime.utcnow().replace(microsecond=0)
self.assertIsNone(
self.master_loop.handle_result_request(task_info, param))
with transaction_cursor(self.connection) as cursor:
cursor.execute('SELECT COUNT(*) FROM scoring_statuscheck')
self.assertEqual(cursor.fetchone()[0], 1)
cursor.execute(
'SELECT status FROM scoring_statuscheck'
' WHERE service_id = 1 AND team_id = 2 AND tick = 1')
self.assertEqual(cursor.fetchone()[0], CheckResult.OK.value)
cursor.execute(
'SELECT placement_end FROM scoring_flag'
' WHERE service_id = 1 AND protecting_team_id = 2 AND tick = 1'
)
self.assertGreaterEqual(cursor.fetchone()[0], start_time)
task_info['tick'] = 2
param = CheckResult.FAULTY.value
start_time = datetime.datetime.utcnow().replace(microsecond=0)
self.assertIsNone(
self.master_loop.handle_result_request(task_info, param))
with transaction_cursor(self.connection) as cursor:
cursor.execute(
'SELECT status FROM scoring_statuscheck'
' WHERE service_id = 1 AND team_id = 2 AND tick = 2')
self.assertEqual(cursor.fetchone()[0], CheckResult.FAULTY.value)
cursor.execute(
'SELECT placement_end FROM scoring_flag'
' WHERE service_id = 1 AND protecting_team_id = 2 AND tick = 2'
)
self.assertGreaterEqual(cursor.fetchone()[0], start_time)
task_info['tick'] = 3
param = 'Not an int'
self.assertIsNone(
self.master_loop.handle_result_request(task_info, param))
with transaction_cursor(self.connection) as cursor:
cursor.execute(
'SELECT status FROM scoring_statuscheck'
' WHERE service_id = 1 AND team_id = 2 AND tick = 3')
self.assertIsNone(cursor.fetchone())
cursor.execute(
'SELECT placement_end FROM scoring_flag'
' WHERE service_id = 1 AND protecting_team_id = 2 AND tick = 3'
)
self.assertIsNone(cursor.fetchone()[0])
param = 1337
self.assertIsNone(
self.master_loop.handle_result_request(task_info, param))
with transaction_cursor(self.connection) as cursor:
cursor.execute(
'SELECT status FROM scoring_statuscheck'
' WHERE service_id = 1 AND team_id = 2 AND tick = 3')
self.assertIsNone(cursor.fetchone())
cursor.execute(
'SELECT placement_end FROM scoring_flag'
' WHERE service_id = 1 AND protecting_team_id = 2 AND tick = 3'
)
self.assertIsNone(cursor.fetchone()[0])
@patch('ctf_gameserver.checker.database.get_check_duration')
def test_update_launch_params(self, check_duration_mock):
# Very short duration, but should be ignored in tick 1
check_duration_mock.return_value = 1
self.master_loop.update_launch_params(-1)
self.assertEqual(self.master_loop.tasks_per_launch, 0)
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=1')
self.master_loop.update_launch_params(1)
self.assertEqual(self.master_loop.tasks_per_launch, 1)
with transaction_cursor(self.connection) as cursor:
for i in range(10, 400):
username = '******'.format(i)
email = '{}@example.org'.format(username)
cursor.execute(
'INSERT INTO auth_user (id, username, first_name, last_name, email, password,'
' is_superuser, is_staff, is_active, date_joined)'
' VALUES (%s, %s, %s, %s, %s, %s, false, false, true, NOW())',
(i, username, '', '', '', 'password'))
cursor.execute(
'INSERT INTO registration_team (user_id, informal_email, image, affiliation,'
' country, nop_team)'
' VALUES (%s, %s, %s, %s, %s, false)',
(i, email, '', '', 'World'))
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, %s, 1)', (i, ))
self.master_loop.update_launch_params(1)
self.assertEqual(self.master_loop.tasks_per_launch, 49)
check_duration_mock.return_value = None
self.master_loop.update_launch_params(10)
self.assertEqual(self.master_loop.tasks_per_launch, 49)
check_duration_mock.return_value = 3600
self.master_loop.update_launch_params(10)
self.assertEqual(self.master_loop.tasks_per_launch, 49)
check_duration_mock.return_value = 90
self.master_loop.update_launch_params(10)
self.assertEqual(self.master_loop.tasks_per_launch, 7)
self.master_loop.interval = 5
self.master_loop.update_launch_params(10)
self.assertEqual(self.master_loop.tasks_per_launch, 4)
check_duration_mock.return_value = 10
self.master_loop.interval = 10
self.master_loop.tick_duration = datetime.timedelta(seconds=90)
self.master_loop.update_launch_params(10)
self.assertEqual(self.master_loop.tasks_per_launch, 7)
def test_sudo_unfinished(self, monotonic_mock, warning_mock):
if shutil.which('sudo') is None or not os.path.exists(
'/etc/sudoers.d/ctf-checker'):
raise SkipTest('sudo or sudo config not available')
checkerscript_path = os.path.join(
os.path.dirname(__file__),
'integration_unfinished_checkerscript.py')
checkerscript_pidfile = tempfile.NamedTemporaryFile()
os.chmod(checkerscript_pidfile.name, 0o666)
os.environ['CHECKERSCRIPT_PIDFILE'] = checkerscript_pidfile.name
monotonic_mock.return_value = 10
master_loop = MasterLoop(self.connection, self.state_db_conn,
'service1', checkerscript_path,
'ctf-checkerrunner', 2, 1, 10, '0.0.%s.1',
b'secret', {}, DummyQueue())
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET start=NOW()')
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=0')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 0)')
monotonic_mock.return_value = 20
master_loop.supervisor.queue_timeout = 0.01
# Checker Script gets started, will return False because no messages yet
self.assertFalse(master_loop.step())
master_loop.supervisor.queue_timeout = 10
self.assertTrue(master_loop.step())
checkerscript_pidfile.seek(0)
checkerscript_pid = int(checkerscript_pidfile.read())
def signal_script():
subprocess.check_call([
'sudo', '--user=ctf-checkerrunner', '--', 'kill', '-0',
str(checkerscript_pid)
])
# Ensure process is running by sending signal 0
signal_script()
master_loop.supervisor.queue_timeout = 0.01
monotonic_mock.return_value = 50
self.assertFalse(master_loop.step())
# Process should still be running
signal_script()
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=1')
monotonic_mock.return_value = 190
self.assertFalse(master_loop.step())
# Poll whether the process has been killed
for _ in range(100):
try:
signal_script()
except subprocess.CalledProcessError:
break
time.sleep(0.1)
with self.assertRaises(subprocess.CalledProcessError):
signal_script()
with transaction_cursor(self.connection) as cursor:
cursor.execute(
'SELECT COUNT(*) FROM scoring_flag'
' WHERE placement_start IS NOT NULL AND placement_end IS NULL'
)
self.assertEqual(cursor.fetchone()[0], 1)
cursor.execute('SELECT COUNT(*) FROM scoring_statuscheck')
self.assertEqual(cursor.fetchone()[0], 0)
warning_mock.assert_called_with('Terminating all %d Runner processes',
1)
del os.environ['CHECKERSCRIPT_PIDFILE']
checkerscript_pidfile.close()
def test_state(self, monotonic_mock):
checkerscript_path = os.path.join(
os.path.dirname(__file__), 'integration_state_checkerscript.py')
monotonic_mock.return_value = 10
master_loop = MasterLoop(self.connection, self.state_db_conn,
'service1', checkerscript_path, None, 2, 1,
10, '0.0.%s.1', b'secret', {}, DummyQueue())
with transaction_cursor(self.state_db_conn) as cursor:
# Prepopulate state for the non-checked service to ensure we'll never get this data returned
data = 'gAN9cQBYAwAAAGZvb3EBWAMAAABiYXJxAnMu'
cursor.execute(
'INSERT INTO checkerstate (team_net_no, service_id, identifier, data)'
' VALUES (92, 2, %s, %s), (93, 2, %s, %s)',
('key1', data, 'key2', data))
# Tick 0
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET start=NOW()')
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=0')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 0), (1, 3, 0)')
monotonic_mock.return_value = 20
master_loop.supervisor.queue_timeout = 0.01
self.assertFalse(master_loop.step())
monotonic_mock.return_value = 100
master_loop.supervisor.queue_timeout = 10
while master_loop.step() or master_loop.get_running_script_count() > 0:
pass
with transaction_cursor(self.connection) as cursor:
cursor.execute('SELECT COUNT(*) FROM scoring_flag'
' WHERE placement_end IS NOT NULL')
self.assertEqual(cursor.fetchone()[0], 2)
cursor.execute(
'SELECT COUNT(*) FROM scoring_statuscheck WHERE status=%s',
(CheckResult.OK.value, ))
self.assertEqual(cursor.fetchone()[0], 2)
# Tick 1
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=1')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 1), (1, 3, 1)')
monotonic_mock.return_value = 200
master_loop.supervisor.queue_timeout = 0.01
self.assertFalse(master_loop.step())
monotonic_mock.return_value = 280
master_loop.supervisor.queue_timeout = 10
while master_loop.step() or master_loop.get_running_script_count() > 0:
pass
with transaction_cursor(self.connection) as cursor:
cursor.execute('SELECT COUNT(*) FROM scoring_flag'
' WHERE placement_end IS NOT NULL')
self.assertEqual(cursor.fetchone()[0], 4)
cursor.execute(
'SELECT COUNT(*) FROM scoring_statuscheck WHERE status=%s',
(CheckResult.OK.value, ))
self.assertEqual(cursor.fetchone()[0], 4)
# Tick 2
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=2')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 2), (1, 3, 2)')
monotonic_mock.return_value = 380
master_loop.supervisor.queue_timeout = 0.01
self.assertFalse(master_loop.step())
monotonic_mock.return_value = 460
master_loop.supervisor.queue_timeout = 10
while master_loop.step() or master_loop.get_running_script_count() > 0:
pass
with transaction_cursor(self.connection) as cursor:
cursor.execute('SELECT COUNT(*) FROM scoring_flag'
' WHERE placement_end IS NOT NULL')
self.assertEqual(cursor.fetchone()[0], 6)
cursor.execute(
'SELECT COUNT(*) FROM scoring_statuscheck WHERE status=%s',
(CheckResult.OK.value, ))
self.assertEqual(cursor.fetchone()[0], 6)
def test_multi_teams_ticks(self, monotonic_mock):
checkerscript_path = os.path.join(
os.path.dirname(__file__), 'integration_multi_checkerscript.py')
monotonic_mock.return_value = 10
master_loop = MasterLoop(self.connection, self.state_db_conn,
'service1', checkerscript_path, None, 2, 1,
10, '0.0.%s.1', b'secret', {}, DummyQueue())
# Tick 0
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET start=NOW()')
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=0')
# Also add flags for service 2 (which does not get checked) to make sure it won't get touched
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 0), (1, 3, 0), (2, 2, 0), (2, 3, 0)')
monotonic_mock.return_value = 20
master_loop.supervisor.queue_timeout = 0.01
self.assertFalse(master_loop.step())
monotonic_mock.return_value = 100
master_loop.supervisor.queue_timeout = 10
while master_loop.step() or master_loop.get_running_script_count() > 0:
pass
with transaction_cursor(self.connection) as cursor:
cursor.execute('SELECT COUNT(*) FROM scoring_flag'
' WHERE placement_end IS NOT NULL')
self.assertEqual(cursor.fetchone()[0], 2)
cursor.execute('SELECT COUNT(*) FROM scoring_statuscheck')
self.assertEqual(cursor.fetchone()[0], 2)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=2 AND tick=0')
self.assertEqual(cursor.fetchone()[0], CheckResult.FAULTY.value)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=3 AND tick=0')
self.assertEqual(cursor.fetchone()[0], CheckResult.OK.value)
# Tick 1
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=1')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 1), (1, 3, 1), (2, 2, 1), (2, 3, 1)')
monotonic_mock.return_value = 200
master_loop.supervisor.queue_timeout = 0.01
self.assertFalse(master_loop.step())
monotonic_mock.return_value = 280
master_loop.supervisor.queue_timeout = 10
while master_loop.step() or master_loop.get_running_script_count() > 0:
pass
with transaction_cursor(self.connection) as cursor:
cursor.execute('SELECT COUNT(*) FROM scoring_flag'
' WHERE placement_end IS NOT NULL')
self.assertEqual(cursor.fetchone()[0], 4)
cursor.execute('SELECT COUNT(*) FROM scoring_statuscheck')
self.assertEqual(cursor.fetchone()[0], 4)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=2 AND tick=0')
self.assertEqual(cursor.fetchone()[0], CheckResult.FAULTY.value)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=3 AND tick=0')
self.assertEqual(cursor.fetchone()[0], CheckResult.OK.value)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=2 AND tick=1')
self.assertEqual(cursor.fetchone()[0], CheckResult.DOWN.value)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=3 AND tick=1')
self.assertEqual(cursor.fetchone()[0], CheckResult.FAULTY.value)
# Tick 2
with transaction_cursor(self.connection) as cursor:
cursor.execute('UPDATE scoring_gamecontrol SET current_tick=2')
cursor.execute(
'INSERT INTO scoring_flag (service_id, protecting_team_id, tick)'
' VALUES (1, 2, 2), (1, 3, 2), (2, 2, 2), (2, 3, 2)')
monotonic_mock.return_value = 380
master_loop.supervisor.queue_timeout = 0.01
self.assertFalse(master_loop.step())
monotonic_mock.return_value = 460
master_loop.supervisor.queue_timeout = 10
while master_loop.step() or master_loop.get_running_script_count() > 0:
pass
with transaction_cursor(self.connection) as cursor:
cursor.execute('SELECT COUNT(*) FROM scoring_flag'
' WHERE placement_end IS NOT NULL')
self.assertEqual(cursor.fetchone()[0], 6)
cursor.execute('SELECT COUNT(*) FROM scoring_statuscheck')
self.assertEqual(cursor.fetchone()[0], 6)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=2 AND tick=2')
self.assertEqual(cursor.fetchone()[0],
CheckResult.RECOVERING.value)
cursor.execute('SELECT status FROM scoring_statuscheck'
' WHERE service_id=1 AND team_id=3 AND tick=2')
self.assertEqual(cursor.fetchone()[0], CheckResult.OK.value)