def test_deck_run_twice(self, mock_resources, mock_probe_ip, mock_input_store, mock_oonireport): mock_probe_ip.geodata['countrycode'] = 'ZZ' mock_probe_ip.lookup.return_value = defer.succeed(None) mock_probe_ip.resolveGeodata.return_value = defer.succeed(None) mock_resources.check_for_update.return_value = defer.succeed(None) mock_input_store.update.return_value = defer.succeed(None) mock_oonireport.upload_all.return_value = defer.succeed(None) mock_director = mock.MagicMock() d = defer.Deferred() with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store): dummy_clock = task.Clock() scheduler_service = SchedulerService( director=mock_director, _reactor=dummy_clock ) scheduler_service.startService() dummy_clock.advance(30) now_time = datetime.utcnow() DT_FRMT = "%Y-%m-%dT%H:%M:%SZ" for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: dstr = datetime.strptime(in_file.read(), DT_FRMT).strftime("%Y-%m-%dT%H") self.assertEqual(dstr, now_time.strftime("%Y-%m-%dT%H")) dummy_clock.advance(30) dummy_clock.advance(30) dummy_clock.advance(30) dummy_clock.advance(30) dummy_clock.advance(30) dummy_clock.advance(30) # Here we pretend they ran yesterday so to re-trigger the daily # tasks for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier), 'w') as out_file: yesterday = (now_time - timedelta(days=1, hours=2)).strftime(DT_FRMT) out_file.write(yesterday) dummy_clock.advance(30) # We check that the run method of the deck was called twice self.mock_deck.run.assert_has_calls([ mock.call(mock_director), mock.call(mock_director) ]) d.callback(None) return d
def __init__(self, web_ui_port): """ If the advanced->disabled_webui is set to true then we will not start the WebUI. """ service.MultiService.__init__(self) director = Director() self.scheduler_service = SchedulerService(director) self.scheduler_service.setServiceParent(self) if not config.advanced.disabled_webui: self.web_ui_service = WebUIService(director, self.scheduler_service, web_ui_port) self.web_ui_service.setServiceParent(self)
def test_disk_quota_cleanup(self, mock_resources, mock_probe_ip, mock_input_store, mock_oonireport): mock_probe_ip.geodata['countrycode'] = 'ZZ' mock_probe_ip.lookup.return_value = defer.succeed(None) mock_probe_ip.resolveGeodata.return_value = defer.succeed(None) mock_resources.check_for_update.return_value = defer.succeed(None) mock_input_store.update.return_value = defer.succeed(None) mock_oonireport.upload_all.return_value = defer.succeed(None) self.config_mock.basic.measurement_quota = '1M' # We create 10MB of measurements self.create_dummy_measurements(count=10, size=1*1024*1024) measurement_count = len(os.listdir(self.measurements_directory)) mock_director = mock.MagicMock() d = defer.Deferred() with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store): dummy_clock = task.Clock() scheduler_service = SchedulerService( director=mock_director, _reactor=dummy_clock ) scheduler_service.startService() dummy_clock.advance(30) now_time = datetime.utcnow() DT_FRMT = "%Y-%m-%dT%H:%M:%SZ" for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: dstr = datetime.strptime(in_file.read(), DT_FRMT).strftime("%Y-%m-%dT%H") self.assertEqual(dstr, now_time.strftime("%Y-%m-%dT%H")) # Ensure there are less measurements than there were at the # beginning new_measurement_count = len(os.listdir(self.measurements_directory)) self.assertGreater(measurement_count, new_measurement_count) d.callback(None) return d
def test_deck_run_twice(self, mock_resources, mock_probe_ip, mock_input_store, mock_oonireport): mock_probe_ip.geodata['countrycode'] = 'ZZ' mock_probe_ip.lookup.return_value = defer.succeed(None) mock_probe_ip.resolveGeodata.return_value = defer.succeed(None) mock_resources.check_for_update.return_value = defer.succeed(None) mock_input_store.update.return_value = defer.succeed(None) mock_oonireport.upload_all.return_value = defer.succeed(None) mock_director = mock.MagicMock() d = defer.Deferred() dummy_clock = task.Clock() class FakeDatetime(datetime): @staticmethod def utcnow(): return datetime(2000, 1, 1, 7, 0, 0) + timedelta(seconds=dummy_clock.seconds()) with mock.patch('ooni.agent.scheduler.deck_store', self.deck_store), \ mock.patch('ooni.agent.scheduler.datetime', FakeDatetime): scheduler_service = SchedulerService(director=mock_director, _reactor=dummy_clock) scheduler_service.startService() dummy_clock.advance(45) # these tasks were run before clock was pumped for t in scheduler_service._scheduled_tasks: self.assertIn(t.schedule, ('@daily', '@hourly')) with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: self.assertEqual(in_file.read(), '2000-01-01T07:00:00Z') # that's leaping clock, it leads to immediate scheduling dummy_clock.advance(24 * 60 * 60) for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z') # nothing happens during an hour dummy_clock.advance(60 * 60 - 46) self.assertEqual(FakeDatetime.utcnow(), datetime(2000, 1, 2, 7, 59, 59)) for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z') # that's ticking clock, it smears the load a bit dummy_clock.pump([1] * 1800) zero, hourly, daily = 0, 0, 0 for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: if t.schedule == '@daily': daily += 1 self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z') elif t.schedule == '@hourly': hourly += 1 # `:[03]0Z` is caused by scheduler resolution & ticking one second a time last_run = in_file.read() self.assertRegexpMatches(last_run, '^2000-01-02T08:0.:[03]0Z$') if last_run == '2000-01-02T08:00:00Z': zero += 1 self.assertGreater(hourly, 0) self.assertGreater(daily, 0) self.assertLess(zero, hourly) self.assertLessEqual(zero, 1) # should ALMOST never happen # leaping to the end of the day dummy_clock.advance((datetime(2000, 1, 2, 23, 59, 59) - FakeDatetime.utcnow()).total_seconds()) for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: if t.schedule == '@daily': self.assertEqual(in_file.read(), '2000-01-02T07:00:45Z') elif t.schedule == '@hourly': self.assertEqual(in_file.read(), '2000-01-02T23:59:59Z') # save ~30% of the testcase runtime while ticking through six hours for t in scheduler_service._scheduled_tasks[:]: if t.schedule == '@hourly': scheduler_service.unschedule(t) # ticking through six hours dummy_clock.pump([random.uniform(0, 120) for i in xrange(6 * 60)]) for t in scheduler_service._scheduled_tasks: with open(os.path.join(self.scheduler_directory, t.identifier)) as in_file: # randomized clock kills 30s resolution of the scheduler self.assertRegexpMatches(in_file.read(), '^2000-01-03T0[012]:..:..Z$') self.assertGreater(FakeDatetime.utcnow(), datetime(2000, 1, 3, 5, 0, 0)) # should be ~6:00 # verify, that double-run does not happen even in case of reseeding (reboot/restart) dummy_clock.advance((datetime(2000, 1, 3, 23, 59, 59) - FakeDatetime.utcnow()).total_seconds()) launches = {} while FakeDatetime.utcnow() < datetime(2000, 1, 4, 6, 0, 0): for t in scheduler_service._scheduled_tasks: with open( os.path.join(self.scheduler_directory, t.identifier)) as in_file: launches.setdefault(t.identifier, set()) launches[t.identifier].add(in_file.read()) self.assertLessEqual(t._smear_coef, 1.0) t._smear_coef = random.random() dummy_clock.advance(random.uniform(0, 120)) self.assertEqual(len(launches), len(scheduler_service._scheduled_tasks)) self.assertEqual({k: len(v) for k, v in launches.iteritems()}, dict.fromkeys(launches.iterkeys(), 2)) # We check that the run method of the deck was called twice # NB: That does NOT check that @daily task was called exactly twice self.mock_deck.run.assert_has_calls([ mock.call(mock_director, from_schedule=True), mock.call(mock_director, from_schedule=True) ]) d.callback(None) return d