def test_process_scope(self, worker_class_mock): original_next_timestamp_method =\ self.cloudkitty_processor.next_timestamp_to_process next_timestamp_mock_method = mock.Mock() worker_mock = mock.Mock() worker_class_mock.return_value = worker_mock original_worker_class = self.cloudkitty_processor.worker_class self.cloudkitty_processor.worker_class = worker_class_mock try: self.cloudkitty_processor.next_timestamp_to_process =\ next_timestamp_mock_method scope_mock = mock.Mock() next_timestamp_mock_method.return_value = tzutils.localized_now() self.cloudkitty_processor.process_scope(scope_mock) next_timestamp_mock_method.assert_has_calls( [mock.call(scope_mock)]) worker_class_mock.assert_has_calls([ mock.call(self.cloudkitty_processor.collector, self.cloudkitty_processor.storage, scope_mock, self.cloudkitty_processor._worker_id) ]) worker_mock.run.assert_called_once() finally: self.cloudkitty_processor.next_timestamp_to_process =\ original_next_timestamp_method self.cloudkitty_processor.worker_class = original_worker_class
def test_execute_worker_processing_scope_already_processed_inactive( self, do_execute_scope_processing_mock, state_manager_is_storage_scope_active_mock, state_manager_get_stage_mock): timestamp_now = tzutils.localized_now() next_timestamp_to_process_mock = mock.Mock() next_timestamp_to_process_mock.return_value = timestamp_now self.worker.next_timestamp_to_process = next_timestamp_to_process_mock state_manager_get_stage_mock.return_value = mock.Mock() state_manager_is_storage_scope_active_mock.return_value = False return_method_value = self.worker.execute_worker_processing() self.assertFalse(return_method_value) state_manager_get_stage_mock.assert_has_calls( [mock.call(self.worker._tenant_id)]) state_manager_is_storage_scope_active_mock.assert_has_calls( [mock.call(self.worker._tenant_id)]) self.assertTrue(next_timestamp_to_process_mock.called) self.assertFalse(do_execute_scope_processing_mock.called)
def test_execute_measurements_rating(self, dataframe_mock): new_data_frame_mock = mock.Mock() dataframe_mock.return_value = new_data_frame_mock processor_mock_1 = mock.Mock() return_processor_1 = mock.Mock() processor_mock_1.obj.process.return_value = return_processor_1 processor_mock_2 = mock.Mock() return_processor_2 = mock.Mock() processor_mock_2.obj.process.return_value = return_processor_2 self.worker._processors = [processor_mock_1, processor_mock_2] start_time = tzutils.localized_now() end_time = start_time + datetime.timedelta(hours=1) return_of_method = self.worker.execute_measurements_rating( end_time, start_time, {}) self.assertEqual(return_processor_2, return_of_method) processor_mock_1.obj.process.assert_has_calls( [mock.call(new_data_frame_mock)]) processor_mock_2.obj.process.assert_has_calls( [mock.call(return_processor_1)]) dataframe_mock.assert_has_calls( [mock.call(start=start_time, end=end_time, usage={})])
def test_do_execute_scope_processing_with_usage_data( self, update_scope_processing_state_db_mock, persist_rating_data_mock, execute_measurements_rating_mock, do_collection_mock): self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}} usage_data_mock = {"some_usage_data": 2} do_collection_mock.return_value = usage_data_mock execute_measurements_rating_mock_return = mock.Mock() execute_measurements_rating_mock.return_value =\ execute_measurements_rating_mock_return timestamp_now = tzutils.localized_now() self.worker.do_execute_scope_processing(timestamp_now) do_collection_mock.assert_has_calls( [mock.call(["metric1", "metric2"], timestamp_now)]) end_time = tzutils.add_delta( timestamp_now, datetime.timedelta(seconds=self.worker._period)) execute_measurements_rating_mock.assert_has_calls( [mock.call(end_time, timestamp_now, usage_data_mock)]) persist_rating_data_mock.assert_has_calls([ mock.call(end_time, execute_measurements_rating_mock_return, timestamp_now) ]) self.assertTrue(update_scope_processing_state_db_mock.called)
def test_next_timestamp_to_process(self, scheduler_db_mock_get_from_db_mock, cloudkitty_processor_init_mock): start_time = tzutils.localized_now() scope = CloudKittyReprocessorTest.create_scope_mock(start_time) scheduler_db_mock_get_from_db_mock.return_value = scope reprocessor = CloudKittyReprocessorTest.create_cloudkitty_reprocessor() next_timestamp = reprocessor._next_timestamp_to_process(scope) expected_calls = [ mock.call(identifier=scope.identifier, start_reprocess_time=scope.start_reprocess_time, end_reprocess_time=scope.end_reprocess_time) ] # There is no current timestamp in the mock object. # Therefore, the next to process is the start timestamp expected_next_timestamp = start_time self.assertEqual(expected_next_timestamp, next_timestamp) cloudkitty_processor_init_mock.assert_called_once() scheduler_db_mock_get_from_db_mock.assert_has_calls(expected_calls)
def test_persist_rating_data(self): start_time = tzutils.localized_now() end_time = start_time + datetime.timedelta(hours=1) frame = {"id": "sd"} self.worker.persist_rating_data(end_time, frame, start_time) self.storage_mock.push.assert_has_calls( [mock.call([frame], self.worker._tenant_id)])
def test_do_execute_scope_processing( self, do_execute_scope_processing_mock_from_worker): now_timestamp = tzutils.localized_now() self.reprocessing_worker.do_execute_scope_processing(now_timestamp) expected_end = tzutils.localized_now() + datetime.timedelta( seconds=self.reprocessing_worker._period) self.storage_mock.delete.assert_has_calls([ mock.call(begin=now_timestamp, end=expected_end, filters={ self.reprocessing_worker.scope_key: self.reprocessing_worker._tenant_id }) ]) do_execute_scope_processing_mock_from_worker.assert_has_calls( [mock.call(now_timestamp)])
def setUp(self): super(TestReprocessSchedulerPostApi, self).setUp() self.endpoint = reprocess.ReprocessSchedulerPostApi() self.scope_ids = [ "some-other-scope-id", "5e56cb64-4980-4466-9fce-d0133c0c221e" ] self.start_reprocess_time = tzutils.localized_now() self.end_reprocess_time =\ self.start_reprocess_time + datetime.timedelta(hours=1) self.reason = "We are testing the reprocess API."
def test_update_scope_processing_state_db(self, update_reprocessing_time_mock): timestamp_now = tzutils.localized_now() self.reprocessing_worker.update_scope_processing_state_db( timestamp_now) start_time = self.reprocessing_worker.scope.start_reprocess_time end_time = self.reprocessing_worker.scope.end_reprocess_time update_reprocessing_time_mock.assert_has_calls([ mock.call(identifier=self.reprocessing_worker.scope.identifier, start_reprocess_time=start_time, end_reprocess_time=end_time, new_current_time_stamp=timestamp_now) ])
def test_do_execute_scope_processing_with_no_usage_data( self, update_scope_processing_state_db_mock, persist_rating_data_mock, execute_measurements_rating_mock, do_collection_mock): self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}} do_collection_mock.return_value = None timestamp_now = tzutils.localized_now() self.worker.do_execute_scope_processing(timestamp_now) do_collection_mock.assert_has_calls( [mock.call(["metric1", "metric2"], timestamp_now)]) self.assertFalse(execute_measurements_rating_mock.called) self.assertFalse(persist_rating_data_mock.called) self.assertTrue(update_scope_processing_state_db_mock.called)
def test_get(self, reprocessing_db_get_all_mock, policy_mock, request_mock): time_now = tzutils.localized_now() schedule_mock = mock.Mock() schedule_mock.id = 1 schedule_mock.identifier = "scope_identifier" schedule_mock.reason = "reason to process" schedule_mock.current_reprocess_time = time_now schedule_mock.start_reprocess_time =\ time_now - datetime.timedelta(hours=10) schedule_mock.end_reprocess_time =\ time_now + datetime.timedelta(hours=10) reprocessing_db_get_all_mock.return_value = [schedule_mock] request_mock.context = mock.Mock() request_mock.args = mock.Mock() request_mock.args.lists = mock.Mock() request_mock.args.lists.return_value = [] list_all_return = self.endpoint.get() self.assertTrue("results" in list_all_return) self.assertTrue("id" not in list_all_return['results'][0]) self.assertTrue("scope_id" in list_all_return['results'][0]) self.assertTrue("reason" in list_all_return['results'][0]) self.assertTrue( "current_reprocess_time" in list_all_return['results'][0]) self.assertTrue( "start_reprocess_time" in list_all_return['results'][0]) self.assertTrue("end_reprocess_time" in list_all_return['results'][0]) self.assertEqual("scope_identifier", list_all_return['results'][0]['scope_id']) self.assertEqual("reason to process", list_all_return['results'][0]['reason']) self.assertEqual( time_now.isoformat(), list_all_return['results'][0]['current_reprocess_time']) self.assertEqual((time_now - datetime.timedelta(hours=10)).isoformat(), list_all_return['results'][0]['start_reprocess_time']) self.assertEqual((time_now + datetime.timedelta(hours=10)).isoformat(), list_all_return['results'][0]['end_reprocess_time']) reprocessing_db_get_all_mock.assert_called_once() policy_mock.assert_called_once()
def setUp(self): super(ReprocessingWorkerTest, self).setUp() patcher_reprocessing_scheduler_db_get_from_db = mock.patch( "cloudkitty.storage_state.ReprocessingSchedulerDb.get_from_db") self.addCleanup(patcher_reprocessing_scheduler_db_get_from_db.stop) self.reprocessing_scheduler_db_get_from_db_mock =\ patcher_reprocessing_scheduler_db_get_from_db.start() patcher_state_manager_get_all = mock.patch( "cloudkitty.storage_state.StateManager.get_all") self.addCleanup(patcher_state_manager_get_all.stop) self.state_manager_get_all_mock = patcher_state_manager_get_all.start() self.collector_mock = mock.Mock() self.storage_mock = mock.Mock() self.scope_key_mock = "key_mock" self.worker_id = 1 self.scope_id = "scope_id1" self.scope_mock = mock.Mock() self.scope_mock.identifier = self.scope_id load_conf_manager = mock.patch("cloudkitty.utils.load_conf") self.addCleanup(load_conf_manager.stop) self.load_conf_mock = load_conf_manager.start() def to_string_scope_mock(self): return "toStringMock" self.scope_mock.__str__ = to_string_scope_mock self.scope_mock.scope_key = self.scope_key_mock self.state_manager_get_all_mock.return_value = [self.scope_mock] self.reprocessing_worker = self.create_reprocessing_worker() self.mock_scheduler = mock.Mock() self.mock_scheduler.identifier = self.scope_id self.start_schedule_mock = tzutils.localized_now() self.mock_scheduler.start_reprocess_time = self.start_schedule_mock self.mock_scheduler.current_reprocess_time = None self.mock_scheduler.end_reprocess_time =\ self.start_schedule_mock + datetime.timedelta(hours=1)
def test_collect_with_data(self): metric = "metric1" timestamp_now = tzutils.localized_now() usage_data = {"some_usage_data": 3} self.collector_mock.retrieve.return_value = (metric, usage_data) return_of_method = self.worker._collect(metric, timestamp_now) next_timestamp = tzutils.add_delta( timestamp_now, datetime.timedelta(seconds=self.worker._period)) self.collector_mock.retrieve.assert_has_calls([ mock.call(metric, timestamp_now, next_timestamp, self.worker._tenant_id) ]) self.assertEqual((metric, usage_data), return_of_method)
def test_check_state(self, check_time_state_mock): state_mock = mock.Mock() timestamp_now = tzutils.localized_now() state_mock._state.get_state.return_value = timestamp_now expected_time = timestamp_now + datetime.timedelta(hours=1) check_time_state_mock.return_value = \ expected_time return_of_method = orchestrator._check_state(state_mock, 3600, self._tenant_id) self.assertEqual(expected_time, return_of_method) state_mock._state.get_state.assert_has_calls( [mock.call(self._tenant_id)]) check_time_state_mock.assert_has_calls( [mock.call(timestamp_now, 3600, 2)])
def test_do_collection_all_valid(self): timestamp_now = tzutils.localized_now() metrics = ['metric{}'.format(i) for i in range(5)] side_effect = [( metrics[i], { 'period': { 'begin': 0, 'end': 3600 }, 'usage': i }, ) for i in range(5)] self.collector_mock.retrieve.side_effect = side_effect output = sorted(self.worker._do_collection(metrics, timestamp_now).items(), key=lambda x: x[1]['usage']) self.assertEqual(side_effect, output)
def update_storage_scope(self, storage_scope_to_update, scope_key=None, fetcher=None, collector=None, active=None): """Update storage scope data. :param storage_scope_to_update: The storage scope to update in the DB :type storage_scope_to_update: object :param fetcher: Fetcher associated to the scope :type fetcher: str :param collector: Collector associated to the scope :type collector: str :param scope_key: scope_key associated to the scope :type scope_key: str :param active: indicates if the storage scope is active for processing :type active: bool """ session = db.get_session() session.begin() db_scope = self._get_db_item(session, storage_scope_to_update.identifier, storage_scope_to_update.fetcher, storage_scope_to_update.collector, storage_scope_to_update.scope_key) if scope_key: db_scope.scope_key = scope_key if fetcher: db_scope.fetcher = fetcher if collector: db_scope.collector = collector if active is not None and active != db_scope.active: db_scope.active = active now = tzutils.localized_now() db_scope.scope_activation_toggle_date = tzutils.local_to_utc( now, naive=True) session.commit() session.close()
def test_collect_no_data(self): metric = "metric1" timestamp_now = tzutils.localized_now() self.collector_mock.retrieve.return_value = (metric, None) expected_message = "Collector 'toString' returned no data for " \ "resource 'metric1'" expected_message = re.escape(expected_message) self.assertRaisesRegex(collector.NoDataCollected, expected_message, self.worker._collect, metric, timestamp_now) next_timestamp = tzutils.add_delta( timestamp_now, datetime.timedelta(seconds=self.worker._period)) self.collector_mock.retrieve.assert_has_calls([ mock.call(metric, timestamp_now, next_timestamp, self.worker._tenant_id) ])
def test_next_timestamp_to_process_processing_finished( self, scheduler_db_mock_get_from_db_mock, cloudkitty_processor_init_mock): start_time = tzutils.localized_now() scope = CloudKittyReprocessorTest.create_scope_mock(start_time) scheduler_db_mock_get_from_db_mock.return_value = None reprocessor = CloudKittyReprocessorTest.create_cloudkitty_reprocessor() next_timestamp = reprocessor._next_timestamp_to_process(scope) expected_calls = [ mock.call(identifier=scope.identifier, start_reprocess_time=scope.start_reprocess_time, end_reprocess_time=scope.end_reprocess_time) ] self.assertIsNone(next_timestamp) cloudkitty_processor_init_mock.assert_called_once() scheduler_db_mock_get_from_db_mock.assert_has_calls(expected_calls)
def quote(self, ctxt, res_data): LOG.debug('Received quote request [%s] from RPC.', res_data) worker = APIWorker() start = tzutils.localized_now() end = tzutils.add_delta(start, timedelta(seconds=CONF.collect.period)) # Need to prepare data to support the V2 processing format usage = {} for k in res_data['usage']: all_data_points_for_metric = [] all_quote_data_entries = res_data['usage'][k] for p in all_quote_data_entries: vol = p['vol'] desc = p.get('desc', {}) data_point = dataframe.DataPoint( vol['unit'], vol['qty'], 0, desc.get('groupby', []), desc.get('metadata', []), ) all_data_points_for_metric.append(data_point) usage[k] = all_data_points_for_metric frame = dataframe.DataFrame( start=start, end=end, usage=usage, ) quote_result = worker.quote(frame) LOG.debug("Quote result [%s] for input data [%s].", quote_result, res_data) return str(quote_result)
def test_do_collection_some_empty(self): timestamp_now = tzutils.localized_now() metrics = ['metric{}'.format(i) for i in range(7)] side_effect = [( metrics[i], { 'period': { 'begin': 0, 'end': 3600 }, 'usage': i }, ) for i in range(5)] side_effect.insert(2, collector.NoDataCollected('a', 'b')) side_effect.insert(4, collector.NoDataCollected('a', 'b')) self.collector_mock.retrieve.side_effect = side_effect output = sorted(self.worker._do_collection(metrics, timestamp_now).items(), key=lambda x: x[1]['usage']) self.assertEqual([ i for i in side_effect if not isinstance(i, collector.NoDataCollected) ], output)
def check_time_state(timestamp=None, period=0, wait_periods=0): """Checks the state of a timestamp compared to the current time. Returns the next timestamp based on the current timestamp and the period if the next timestamp is inferior to the current time and the waiting period or None if not. :param timestamp: Current timestamp :type timestamp: datetime.datetime :param period: Period, in seconds :type period: int :param wait_periods: periods to wait before the current timestamp. :type wait_periods: int :rtype: datetime.datetime """ if not timestamp: return tzutils.get_month_start() period_delta = datetime.timedelta(seconds=period) next_timestamp = tzutils.add_delta(timestamp, period_delta) wait_time = wait_periods * period_delta if tzutils.add_delta(next_timestamp, wait_time) < tzutils.localized_now(): return next_timestamp return None
def initialize_data(self): data = test_utils.generate_v2_storage_data( start=tzutils.get_month_start(), end=tzutils.localized_now().replace(hour=0), ) self.storage.push([data])
def test_update_scope_processing_state_db(self): timestamp = tzutils.localized_now() self.worker.update_scope_processing_state_db(timestamp) self.state_manager_set_state_mock.assert_has_calls( [mock.call(self.worker._tenant_id, timestamp)])
def setUp(self): self.local_now = tzutils.localized_now() self.naive_now = utils.utcnow().replace(microsecond=0)