class AgentService(service.MultiService):
"""Manage all services related to the ooniprobe-agent daemon."""
def __init__(self, web_ui_port):
"""
If the advanced->disabled_webui is set to true then we will not start the WebUI.
"""
service.MultiService.__init__(self)
director = Director()
self.scheduler_service = SchedulerService(director)
self.scheduler_service.setServiceParent(self)
if not config.advanced.disabled_webui:
self.web_ui_service = WebUIService(director,
self.scheduler_service,
web_ui_port)
self.web_ui_service.setServiceParent(self)
def startService(self):
service.MultiService.startService(self)
def stopService(self):
service.MultiService.stopService(self)
def test_deck_run_twice(self, mock_resources, mock_probe_ip,
mock_input_store, mock_oonireport):
mock_probe_ip.geodata['countrycode'] = 'ZZ'
mock_probe_ip.lookup.return_value = defer.succeed(None)
mock_probe_ip.resolveGeodata.return_value = defer.succeed(None)
mock_resources.check_for_update.return_value = defer.succeed(None)
mock_input_store.update.return_value = defer.succeed(None)
mock_oonireport.upload_all.return_value = defer.succeed(None)
mock_director = mock.MagicMock()
d = defer.Deferred()
with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store):
dummy_clock = task.Clock()
scheduler_service = SchedulerService(
director=mock_director,
_reactor=dummy_clock
)
scheduler_service.startService()
dummy_clock.advance(30)
now_time = datetime.utcnow()
DT_FRMT = "%Y-%m-%dT%H:%M:%SZ"
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
dstr = datetime.strptime(in_file.read(),
DT_FRMT).strftime("%Y-%m-%dT%H")
self.assertEqual(dstr, now_time.strftime("%Y-%m-%dT%H"))
dummy_clock.advance(30)
dummy_clock.advance(30)
dummy_clock.advance(30)
dummy_clock.advance(30)
dummy_clock.advance(30)
dummy_clock.advance(30)
# Here we pretend they ran yesterday so to re-trigger the daily
# tasks
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier), 'w') as out_file:
yesterday = (now_time - timedelta(days=1,
hours=2)).strftime(DT_FRMT)
out_file.write(yesterday)
dummy_clock.advance(30)
# We check that the run method of the deck was called twice
self.mock_deck.run.assert_has_calls([
mock.call(mock_director), mock.call(mock_director)
])
d.callback(None)
return d
def __init__(self, web_ui_port):
"""
If the advanced->disabled_webui is set to true then we will not start the WebUI.
"""
service.MultiService.__init__(self)
director = Director()
self.scheduler_service = SchedulerService(director)
self.scheduler_service.setServiceParent(self)
if not config.advanced.disabled_webui:
self.web_ui_service = WebUIService(director,
self.scheduler_service,
web_ui_port)
self.web_ui_service.setServiceParent(self)
def test_disk_quota_cleanup(self, mock_resources, mock_probe_ip,
mock_input_store, mock_oonireport):
mock_probe_ip.geodata['countrycode'] = 'ZZ'
mock_probe_ip.lookup.return_value = defer.succeed(None)
mock_probe_ip.resolveGeodata.return_value = defer.succeed(None)
mock_resources.check_for_update.return_value = defer.succeed(None)
mock_input_store.update.return_value = defer.succeed(None)
mock_oonireport.upload_all.return_value = defer.succeed(None)
self.config_mock.basic.measurement_quota = '1M'
# We create 10MB of measurements
self.create_dummy_measurements(count=10, size=1*1024*1024)
measurement_count = len(os.listdir(self.measurements_directory))
mock_director = mock.MagicMock()
d = defer.Deferred()
with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store):
dummy_clock = task.Clock()
scheduler_service = SchedulerService(
director=mock_director,
_reactor=dummy_clock
)
scheduler_service.startService()
dummy_clock.advance(30)
now_time = datetime.utcnow()
DT_FRMT = "%Y-%m-%dT%H:%M:%SZ"
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
dstr = datetime.strptime(in_file.read(),
DT_FRMT).strftime("%Y-%m-%dT%H")
self.assertEqual(dstr, now_time.strftime("%Y-%m-%dT%H"))
# Ensure there are less measurements than there were at the
# beginning
new_measurement_count = len(os.listdir(self.measurements_directory))
self.assertGreater(measurement_count, new_measurement_count)
d.callback(None)
return d
def test_disk_quota_cleanup(self, mock_resources, mock_probe_ip,
mock_input_store, mock_oonireport):
mock_probe_ip.geodata['countrycode'] = 'ZZ'
mock_probe_ip.lookup.return_value = defer.succeed(None)
mock_probe_ip.resolveGeodata.return_value = defer.succeed(None)
mock_resources.check_for_update.return_value = defer.succeed(None)
mock_input_store.update.return_value = defer.succeed(None)
mock_oonireport.upload_all.return_value = defer.succeed(None)
self.config_mock.basic.measurement_quota = '1M'
# We create 10MB of measurements
self.create_dummy_measurements(count=10, size=1*1024*1024)
measurement_count = len(os.listdir(self.measurements_directory))
mock_director = mock.MagicMock()
d = defer.Deferred()
with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store):
dummy_clock = task.Clock()
scheduler_service = SchedulerService(
director=mock_director,
_reactor=dummy_clock
)
scheduler_service.startService()
dummy_clock.advance(30)
now_time = datetime.utcnow()
DT_FRMT = "%Y-%m-%dT%H:%M:%SZ"
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
dstr = datetime.strptime(in_file.read(),
DT_FRMT).strftime("%Y-%m-%dT%H")
self.assertEqual(dstr, now_time.strftime("%Y-%m-%dT%H"))
# Ensure there are less measurements than there were at the
# beginning
new_measurement_count = len(os.listdir(self.measurements_directory))
self.assertGreater(measurement_count, new_measurement_count)
d.callback(None)
return d
class AgentService(service.MultiService):
"""Manage all services related to the ooniprobe-agent daemon."""
def __init__(self, web_ui_port):
"""
If the advanced->disabled_webui is set to true then we will not start the WebUI.
"""
service.MultiService.__init__(self)
director = Director()
self.scheduler_service = SchedulerService(director)
self.scheduler_service.setServiceParent(self)
if not config.advanced.disabled_webui:
self.web_ui_service = WebUIService(director,
self.scheduler_service,
web_ui_port)
self.web_ui_service.setServiceParent(self)
def startService(self):
service.MultiService.startService(self)
def stopService(self):
service.MultiService.stopService(self)
def __init__(self, web_ui_port):
"""
If the advanced->disabled_webui is set to true then we will not start the WebUI.
"""
service.MultiService.__init__(self)
director = Director()
self.scheduler_service = SchedulerService(director)
self.scheduler_service.setServiceParent(self)
if not config.advanced.disabled_webui:
self.web_ui_service = WebUIService(director,
self.scheduler_service,
web_ui_port)
self.web_ui_service.setServiceParent(self)
def test_deck_run_twice(self, mock_resources, mock_probe_ip,
mock_input_store, mock_oonireport):
mock_probe_ip.geodata['countrycode'] = 'ZZ'
mock_probe_ip.lookup.return_value = defer.succeed(None)
mock_probe_ip.resolveGeodata.return_value = defer.succeed(None)
mock_resources.check_for_update.return_value = defer.succeed(None)
mock_input_store.update.return_value = defer.succeed(None)
mock_oonireport.upload_all.return_value = defer.succeed(None)
mock_director = mock.MagicMock()
d = defer.Deferred()
dummy_clock = task.Clock()
class FakeDatetime(datetime):
@staticmethod
def utcnow():
return datetime(2000,1,1, 7,0,0) + timedelta(seconds=dummy_clock.seconds())
with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store), \
mock.patch('ooni.agent.scheduler.datetime', FakeDatetime):
scheduler_service = SchedulerService(
director=mock_director,
_reactor=dummy_clock
)
scheduler_service.startService()
dummy_clock.advance(45)
# these tasks were run before clock was pumped
for t in scheduler_service._scheduled_tasks:
self.assertIn(t.schedule, ('@daily', '@hourly'))
with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file:
self.assertEqual(in_file.read(), '2000-01-01T07:00:00Z')
# that's leaping clock, it leads to immediate scheduling
dummy_clock.advance(24 * 60 * 60)
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file:
self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z')
# nothing happens during an hour
dummy_clock.advance(60 * 60 - 46)
self.assertEqual(FakeDatetime.utcnow(), datetime(2000,1,2, 7,59,59))
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file:
self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z')
# that's ticking clock, it smears the load a bit
dummy_clock.pump([1] * 1800)
zero, hourly, daily = 0, 0, 0
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file:
if t.schedule == '@daily':
daily += 1
self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z')
elif t.schedule == '@hourly':
hourly += 1
# `:[03]0Z` is caused by scheduler resolution & ticking one second a time
last_run = in_file.read()
self.assertRegexpMatches(last_run, '^2000-01-02T08:0.:[03]0Z$')
if last_run == '2000-01-02T08:00:00Z':
zero += 1
self.assertGreater(hourly, 0)
self.assertGreater(daily, 0)
self.assertLess(zero, hourly)
self.assertLessEqual(zero, 1) # should ALMOST never happen
# leaping to the end of the day
dummy_clock.advance((datetime(2000,1,2, 23,59,59) - FakeDatetime.utcnow()).total_seconds())
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file:
if t.schedule == '@daily':
self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z')
elif t.schedule == '@hourly':
self.assertEqual(in_file.read(), '2000-01-02T23:59:59Z')
# save ~30% of the testcase runtime while ticking through six hours
for t in scheduler_service._scheduled_tasks[:]:
if t.schedule == '@hourly':
scheduler_service.unschedule(t)
# ticking through six hours
dummy_clock.pump([random.uniform(0, 120) for i in xrange(6*60)])
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file:
# randomized clock kills 30s resolution of the scheduler
self.assertRegexpMatches(in_file.read(), '^2000-01-03T0[012]:..:..Z$')
self.assertGreater(FakeDatetime.utcnow(), datetime(2000,1,3, 5,0,0)) # should be ~6:00
# verify, that double-run does not happen even in case of reseeding (reboot/restart)
dummy_clock.advance((datetime(2000,1,3, 23,59,59) - FakeDatetime.utcnow()).total_seconds())
launches = {}
while FakeDatetime.utcnow() < datetime(2000,1,4, 6,0,0):
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file:
launches.setdefault(t.identifier, set())
launches[t.identifier].add(in_file.read())
self.assertLessEqual(t._smear_coef, 1.0)
t._smear_coef = random.random()
dummy_clock.advance(random.uniform(0, 120))
self.assertEqual(len(launches), len(scheduler_service._scheduled_tasks))
self.assertEqual({k: len(v) for k, v in launches.iteritems()}, dict.fromkeys(launches.iterkeys(), 2))
# We check that the run method of the deck was called twice
# NB: That does NOT check that @daily task was called exactly twice
self.mock_deck.run.assert_has_calls([
mock.call(mock_director, from_schedule=True), mock.call(mock_director, from_schedule=True)
])
d.callback(None)
return d
def test_deck_run_twice(self, mock_resources, mock_probe_ip,
mock_input_store, mock_oonireport):
mock_probe_ip.geodata['countrycode'] = 'ZZ'
mock_probe_ip.lookup.return_value = defer.succeed(None)
mock_probe_ip.resolveGeodata.return_value = defer.succeed(None)
mock_resources.check_for_update.return_value = defer.succeed(None)
mock_input_store.update.return_value = defer.succeed(None)
mock_oonireport.upload_all.return_value = defer.succeed(None)
mock_director = mock.MagicMock()
d = defer.Deferred()
dummy_clock = task.Clock()
class FakeDatetime(datetime):
@staticmethod
def utcnow():
return datetime(2000, 1, 1, 7, 0,
0) + timedelta(seconds=dummy_clock.seconds())
with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store), \
mock.patch('ooni.agent.scheduler.datetime', FakeDatetime):
scheduler_service = SchedulerService(director=mock_director,
_reactor=dummy_clock)
scheduler_service.startService()
dummy_clock.advance(45)
# these tasks were run before clock was pumped
for t in scheduler_service._scheduled_tasks:
self.assertIn(t.schedule, ('@daily', '@hourly'))
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
self.assertEqual(in_file.read(), '2000-01-01T07:00:00Z')
# that's leaping clock, it leads to immediate scheduling
dummy_clock.advance(24 * 60 * 60)
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z')
# nothing happens during an hour
dummy_clock.advance(60 * 60 - 46)
self.assertEqual(FakeDatetime.utcnow(),
datetime(2000, 1, 2, 7, 59, 59))
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z')
# that's ticking clock, it smears the load a bit
dummy_clock.pump([1] * 1800)
zero, hourly, daily = 0, 0, 0
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
if t.schedule == '@daily':
daily += 1
self.assertEqual(in_file.read(),
'2000-01-02T07:00:45Z')
elif t.schedule == '@hourly':
hourly += 1
# `:[03]0Z` is caused by scheduler resolution & ticking one second a time
last_run = in_file.read()
self.assertRegexpMatches(last_run,
'^2000-01-02T08:0.:[03]0Z$')
if last_run == '2000-01-02T08:00:00Z':
zero += 1
self.assertGreater(hourly, 0)
self.assertGreater(daily, 0)
self.assertLess(zero, hourly)
self.assertLessEqual(zero, 1) # should ALMOST never happen
# leaping to the end of the day
dummy_clock.advance((datetime(2000, 1, 2, 23, 59, 59) -
FakeDatetime.utcnow()).total_seconds())
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
if t.schedule == '@daily':
self.assertEqual(in_file.read(),
'2000-01-02T07:00:45Z')
elif t.schedule == '@hourly':
self.assertEqual(in_file.read(),
'2000-01-02T23:59:59Z')
# save ~30% of the testcase runtime while ticking through six hours
for t in scheduler_service._scheduled_tasks[:]:
if t.schedule == '@hourly':
scheduler_service.unschedule(t)
# ticking through six hours
dummy_clock.pump([random.uniform(0, 120) for i in xrange(6 * 60)])
for t in scheduler_service._scheduled_tasks:
with open(os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
# randomized clock kills 30s resolution of the scheduler
self.assertRegexpMatches(in_file.read(),
'^2000-01-03T0[012]:..:..Z$')
self.assertGreater(FakeDatetime.utcnow(),
datetime(2000, 1, 3, 5, 0,
0)) # should be ~6:00
# verify, that double-run does not happen even in case of reseeding (reboot/restart)
dummy_clock.advance((datetime(2000, 1, 3, 23, 59, 59) -
FakeDatetime.utcnow()).total_seconds())
launches = {}
while FakeDatetime.utcnow() < datetime(2000, 1, 4, 6, 0, 0):
for t in scheduler_service._scheduled_tasks:
with open(
os.path.join(self.scheduler_directory,
t.identifier)) as in_file:
launches.setdefault(t.identifier, set())
launches[t.identifier].add(in_file.read())
self.assertLessEqual(t._smear_coef, 1.0)
t._smear_coef = random.random()
dummy_clock.advance(random.uniform(0, 120))
self.assertEqual(len(launches),
len(scheduler_service._scheduled_tasks))
self.assertEqual({k: len(v)
for k, v in launches.iteritems()},
dict.fromkeys(launches.iterkeys(), 2))
# We check that the run method of the deck was called twice
# NB: That does NOT check that @daily task was called exactly twice
self.mock_deck.run.assert_has_calls([
mock.call(mock_director, from_schedule=True),
mock.call(mock_director, from_schedule=True)
])
d.callback(None)
return d