def test_process_scope(self, worker_class_mock):
original_next_timestamp_method =\
self.cloudkitty_processor.next_timestamp_to_process
next_timestamp_mock_method = mock.Mock()
worker_mock = mock.Mock()
worker_class_mock.return_value = worker_mock
original_worker_class = self.cloudkitty_processor.worker_class
self.cloudkitty_processor.worker_class = worker_class_mock
try:
self.cloudkitty_processor.next_timestamp_to_process =\
next_timestamp_mock_method
scope_mock = mock.Mock()
next_timestamp_mock_method.return_value = tzutils.localized_now()
self.cloudkitty_processor.process_scope(scope_mock)
next_timestamp_mock_method.assert_has_calls(
[mock.call(scope_mock)])
worker_class_mock.assert_has_calls([
mock.call(self.cloudkitty_processor.collector,
self.cloudkitty_processor.storage, scope_mock,
self.cloudkitty_processor._worker_id)
])
worker_mock.run.assert_called_once()
finally:
self.cloudkitty_processor.next_timestamp_to_process =\
original_next_timestamp_method
self.cloudkitty_processor.worker_class = original_worker_class
def test_execute_worker_processing_scope_already_processed_inactive(
self, do_execute_scope_processing_mock,
state_manager_is_storage_scope_active_mock,
state_manager_get_stage_mock):
timestamp_now = tzutils.localized_now()
next_timestamp_to_process_mock = mock.Mock()
next_timestamp_to_process_mock.return_value = timestamp_now
self.worker.next_timestamp_to_process = next_timestamp_to_process_mock
state_manager_get_stage_mock.return_value = mock.Mock()
state_manager_is_storage_scope_active_mock.return_value = False
return_method_value = self.worker.execute_worker_processing()
self.assertFalse(return_method_value)
state_manager_get_stage_mock.assert_has_calls(
[mock.call(self.worker._tenant_id)])
state_manager_is_storage_scope_active_mock.assert_has_calls(
[mock.call(self.worker._tenant_id)])
self.assertTrue(next_timestamp_to_process_mock.called)
self.assertFalse(do_execute_scope_processing_mock.called)
def test_execute_measurements_rating(self, dataframe_mock):
new_data_frame_mock = mock.Mock()
dataframe_mock.return_value = new_data_frame_mock
processor_mock_1 = mock.Mock()
return_processor_1 = mock.Mock()
processor_mock_1.obj.process.return_value = return_processor_1
processor_mock_2 = mock.Mock()
return_processor_2 = mock.Mock()
processor_mock_2.obj.process.return_value = return_processor_2
self.worker._processors = [processor_mock_1, processor_mock_2]
start_time = tzutils.localized_now()
end_time = start_time + datetime.timedelta(hours=1)
return_of_method = self.worker.execute_measurements_rating(
end_time, start_time, {})
self.assertEqual(return_processor_2, return_of_method)
processor_mock_1.obj.process.assert_has_calls(
[mock.call(new_data_frame_mock)])
processor_mock_2.obj.process.assert_has_calls(
[mock.call(return_processor_1)])
dataframe_mock.assert_has_calls(
[mock.call(start=start_time, end=end_time, usage={})])
def test_do_execute_scope_processing_with_usage_data(
self, update_scope_processing_state_db_mock,
persist_rating_data_mock, execute_measurements_rating_mock,
do_collection_mock):
self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}}
usage_data_mock = {"some_usage_data": 2}
do_collection_mock.return_value = usage_data_mock
execute_measurements_rating_mock_return = mock.Mock()
execute_measurements_rating_mock.return_value =\
execute_measurements_rating_mock_return
timestamp_now = tzutils.localized_now()
self.worker.do_execute_scope_processing(timestamp_now)
do_collection_mock.assert_has_calls(
[mock.call(["metric1", "metric2"], timestamp_now)])
end_time = tzutils.add_delta(
timestamp_now, datetime.timedelta(seconds=self.worker._period))
execute_measurements_rating_mock.assert_has_calls(
[mock.call(end_time, timestamp_now, usage_data_mock)])
persist_rating_data_mock.assert_has_calls([
mock.call(end_time, execute_measurements_rating_mock_return,
timestamp_now)
])
self.assertTrue(update_scope_processing_state_db_mock.called)
def test_next_timestamp_to_process(self,
scheduler_db_mock_get_from_db_mock,
cloudkitty_processor_init_mock):
start_time = tzutils.localized_now()
scope = CloudKittyReprocessorTest.create_scope_mock(start_time)
scheduler_db_mock_get_from_db_mock.return_value = scope
reprocessor = CloudKittyReprocessorTest.create_cloudkitty_reprocessor()
next_timestamp = reprocessor._next_timestamp_to_process(scope)
expected_calls = [
mock.call(identifier=scope.identifier,
start_reprocess_time=scope.start_reprocess_time,
end_reprocess_time=scope.end_reprocess_time)
]
# There is no current timestamp in the mock object.
# Therefore, the next to process is the start timestamp
expected_next_timestamp = start_time
self.assertEqual(expected_next_timestamp, next_timestamp)
cloudkitty_processor_init_mock.assert_called_once()
scheduler_db_mock_get_from_db_mock.assert_has_calls(expected_calls)
def test_persist_rating_data(self):
start_time = tzutils.localized_now()
end_time = start_time + datetime.timedelta(hours=1)
frame = {"id": "sd"}
self.worker.persist_rating_data(end_time, frame, start_time)
self.storage_mock.push.assert_has_calls(
[mock.call([frame], self.worker._tenant_id)])
def test_do_execute_scope_processing(
self, do_execute_scope_processing_mock_from_worker):
now_timestamp = tzutils.localized_now()
self.reprocessing_worker.do_execute_scope_processing(now_timestamp)
expected_end = tzutils.localized_now() + datetime.timedelta(
seconds=self.reprocessing_worker._period)
self.storage_mock.delete.assert_has_calls([
mock.call(begin=now_timestamp,
end=expected_end,
filters={
self.reprocessing_worker.scope_key:
self.reprocessing_worker._tenant_id
})
])
do_execute_scope_processing_mock_from_worker.assert_has_calls(
[mock.call(now_timestamp)])
def setUp(self):
super(TestReprocessSchedulerPostApi, self).setUp()
self.endpoint = reprocess.ReprocessSchedulerPostApi()
self.scope_ids = [
"some-other-scope-id", "5e56cb64-4980-4466-9fce-d0133c0c221e"
]
self.start_reprocess_time = tzutils.localized_now()
self.end_reprocess_time =\
self.start_reprocess_time + datetime.timedelta(hours=1)
self.reason = "We are testing the reprocess API."
def test_update_scope_processing_state_db(self,
update_reprocessing_time_mock):
timestamp_now = tzutils.localized_now()
self.reprocessing_worker.update_scope_processing_state_db(
timestamp_now)
start_time = self.reprocessing_worker.scope.start_reprocess_time
end_time = self.reprocessing_worker.scope.end_reprocess_time
update_reprocessing_time_mock.assert_has_calls([
mock.call(identifier=self.reprocessing_worker.scope.identifier,
start_reprocess_time=start_time,
end_reprocess_time=end_time,
new_current_time_stamp=timestamp_now)
])
def test_do_execute_scope_processing_with_no_usage_data(
self, update_scope_processing_state_db_mock,
persist_rating_data_mock, execute_measurements_rating_mock,
do_collection_mock):
self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}}
do_collection_mock.return_value = None
timestamp_now = tzutils.localized_now()
self.worker.do_execute_scope_processing(timestamp_now)
do_collection_mock.assert_has_calls(
[mock.call(["metric1", "metric2"], timestamp_now)])
self.assertFalse(execute_measurements_rating_mock.called)
self.assertFalse(persist_rating_data_mock.called)
self.assertTrue(update_scope_processing_state_db_mock.called)
def test_get(self, reprocessing_db_get_all_mock, policy_mock,
request_mock):
time_now = tzutils.localized_now()
schedule_mock = mock.Mock()
schedule_mock.id = 1
schedule_mock.identifier = "scope_identifier"
schedule_mock.reason = "reason to process"
schedule_mock.current_reprocess_time = time_now
schedule_mock.start_reprocess_time =\
time_now - datetime.timedelta(hours=10)
schedule_mock.end_reprocess_time =\
time_now + datetime.timedelta(hours=10)
reprocessing_db_get_all_mock.return_value = [schedule_mock]
request_mock.context = mock.Mock()
request_mock.args = mock.Mock()
request_mock.args.lists = mock.Mock()
request_mock.args.lists.return_value = []
list_all_return = self.endpoint.get()
self.assertTrue("results" in list_all_return)
self.assertTrue("id" not in list_all_return['results'][0])
self.assertTrue("scope_id" in list_all_return['results'][0])
self.assertTrue("reason" in list_all_return['results'][0])
self.assertTrue(
"current_reprocess_time" in list_all_return['results'][0])
self.assertTrue(
"start_reprocess_time" in list_all_return['results'][0])
self.assertTrue("end_reprocess_time" in list_all_return['results'][0])
self.assertEqual("scope_identifier",
list_all_return['results'][0]['scope_id'])
self.assertEqual("reason to process",
list_all_return['results'][0]['reason'])
self.assertEqual(
time_now.isoformat(),
list_all_return['results'][0]['current_reprocess_time'])
self.assertEqual((time_now - datetime.timedelta(hours=10)).isoformat(),
list_all_return['results'][0]['start_reprocess_time'])
self.assertEqual((time_now + datetime.timedelta(hours=10)).isoformat(),
list_all_return['results'][0]['end_reprocess_time'])
reprocessing_db_get_all_mock.assert_called_once()
policy_mock.assert_called_once()
def setUp(self):
super(ReprocessingWorkerTest, self).setUp()
patcher_reprocessing_scheduler_db_get_from_db = mock.patch(
"cloudkitty.storage_state.ReprocessingSchedulerDb.get_from_db")
self.addCleanup(patcher_reprocessing_scheduler_db_get_from_db.stop)
self.reprocessing_scheduler_db_get_from_db_mock =\
patcher_reprocessing_scheduler_db_get_from_db.start()
patcher_state_manager_get_all = mock.patch(
"cloudkitty.storage_state.StateManager.get_all")
self.addCleanup(patcher_state_manager_get_all.stop)
self.state_manager_get_all_mock = patcher_state_manager_get_all.start()
self.collector_mock = mock.Mock()
self.storage_mock = mock.Mock()
self.scope_key_mock = "key_mock"
self.worker_id = 1
self.scope_id = "scope_id1"
self.scope_mock = mock.Mock()
self.scope_mock.identifier = self.scope_id
load_conf_manager = mock.patch("cloudkitty.utils.load_conf")
self.addCleanup(load_conf_manager.stop)
self.load_conf_mock = load_conf_manager.start()
def to_string_scope_mock(self):
return "toStringMock"
self.scope_mock.__str__ = to_string_scope_mock
self.scope_mock.scope_key = self.scope_key_mock
self.state_manager_get_all_mock.return_value = [self.scope_mock]
self.reprocessing_worker = self.create_reprocessing_worker()
self.mock_scheduler = mock.Mock()
self.mock_scheduler.identifier = self.scope_id
self.start_schedule_mock = tzutils.localized_now()
self.mock_scheduler.start_reprocess_time = self.start_schedule_mock
self.mock_scheduler.current_reprocess_time = None
self.mock_scheduler.end_reprocess_time =\
self.start_schedule_mock + datetime.timedelta(hours=1)
def test_collect_with_data(self):
metric = "metric1"
timestamp_now = tzutils.localized_now()
usage_data = {"some_usage_data": 3}
self.collector_mock.retrieve.return_value = (metric, usage_data)
return_of_method = self.worker._collect(metric, timestamp_now)
next_timestamp = tzutils.add_delta(
timestamp_now, datetime.timedelta(seconds=self.worker._period))
self.collector_mock.retrieve.assert_has_calls([
mock.call(metric, timestamp_now, next_timestamp,
self.worker._tenant_id)
])
self.assertEqual((metric, usage_data), return_of_method)
def test_check_state(self, check_time_state_mock):
state_mock = mock.Mock()
timestamp_now = tzutils.localized_now()
state_mock._state.get_state.return_value = timestamp_now
expected_time = timestamp_now + datetime.timedelta(hours=1)
check_time_state_mock.return_value = \
expected_time
return_of_method = orchestrator._check_state(state_mock, 3600,
self._tenant_id)
self.assertEqual(expected_time, return_of_method)
state_mock._state.get_state.assert_has_calls(
[mock.call(self._tenant_id)])
check_time_state_mock.assert_has_calls(
[mock.call(timestamp_now, 3600, 2)])
def test_do_collection_all_valid(self):
timestamp_now = tzutils.localized_now()
metrics = ['metric{}'.format(i) for i in range(5)]
side_effect = [(
metrics[i],
{
'period': {
'begin': 0,
'end': 3600
},
'usage': i
},
) for i in range(5)]
self.collector_mock.retrieve.side_effect = side_effect
output = sorted(self.worker._do_collection(metrics,
timestamp_now).items(),
key=lambda x: x[1]['usage'])
self.assertEqual(side_effect, output)
def update_storage_scope(self,
storage_scope_to_update,
scope_key=None,
fetcher=None,
collector=None,
active=None):
"""Update storage scope data.
:param storage_scope_to_update: The storage scope to update in the DB
:type storage_scope_to_update: object
:param fetcher: Fetcher associated to the scope
:type fetcher: str
:param collector: Collector associated to the scope
:type collector: str
:param scope_key: scope_key associated to the scope
:type scope_key: str
:param active: indicates if the storage scope is active for processing
:type active: bool
"""
session = db.get_session()
session.begin()
db_scope = self._get_db_item(session,
storage_scope_to_update.identifier,
storage_scope_to_update.fetcher,
storage_scope_to_update.collector,
storage_scope_to_update.scope_key)
if scope_key:
db_scope.scope_key = scope_key
if fetcher:
db_scope.fetcher = fetcher
if collector:
db_scope.collector = collector
if active is not None and active != db_scope.active:
db_scope.active = active
now = tzutils.localized_now()
db_scope.scope_activation_toggle_date = tzutils.local_to_utc(
now, naive=True)
session.commit()
session.close()
def test_collect_no_data(self):
metric = "metric1"
timestamp_now = tzutils.localized_now()
self.collector_mock.retrieve.return_value = (metric, None)
expected_message = "Collector 'toString' returned no data for " \
"resource 'metric1'"
expected_message = re.escape(expected_message)
self.assertRaisesRegex(collector.NoDataCollected, expected_message,
self.worker._collect, metric, timestamp_now)
next_timestamp = tzutils.add_delta(
timestamp_now, datetime.timedelta(seconds=self.worker._period))
self.collector_mock.retrieve.assert_has_calls([
mock.call(metric, timestamp_now, next_timestamp,
self.worker._tenant_id)
])
def test_next_timestamp_to_process_processing_finished(
self, scheduler_db_mock_get_from_db_mock,
cloudkitty_processor_init_mock):
start_time = tzutils.localized_now()
scope = CloudKittyReprocessorTest.create_scope_mock(start_time)
scheduler_db_mock_get_from_db_mock.return_value = None
reprocessor = CloudKittyReprocessorTest.create_cloudkitty_reprocessor()
next_timestamp = reprocessor._next_timestamp_to_process(scope)
expected_calls = [
mock.call(identifier=scope.identifier,
start_reprocess_time=scope.start_reprocess_time,
end_reprocess_time=scope.end_reprocess_time)
]
self.assertIsNone(next_timestamp)
cloudkitty_processor_init_mock.assert_called_once()
scheduler_db_mock_get_from_db_mock.assert_has_calls(expected_calls)
def quote(self, ctxt, res_data):
LOG.debug('Received quote request [%s] from RPC.', res_data)
worker = APIWorker()
start = tzutils.localized_now()
end = tzutils.add_delta(start, timedelta(seconds=CONF.collect.period))
# Need to prepare data to support the V2 processing format
usage = {}
for k in res_data['usage']:
all_data_points_for_metric = []
all_quote_data_entries = res_data['usage'][k]
for p in all_quote_data_entries:
vol = p['vol']
desc = p.get('desc', {})
data_point = dataframe.DataPoint(
vol['unit'],
vol['qty'],
0,
desc.get('groupby', []),
desc.get('metadata', []),
)
all_data_points_for_metric.append(data_point)
usage[k] = all_data_points_for_metric
frame = dataframe.DataFrame(
start=start,
end=end,
usage=usage,
)
quote_result = worker.quote(frame)
LOG.debug("Quote result [%s] for input data [%s].",
quote_result, res_data)
return str(quote_result)
def test_do_collection_some_empty(self):
timestamp_now = tzutils.localized_now()
metrics = ['metric{}'.format(i) for i in range(7)]
side_effect = [(
metrics[i],
{
'period': {
'begin': 0,
'end': 3600
},
'usage': i
},
) for i in range(5)]
side_effect.insert(2, collector.NoDataCollected('a', 'b'))
side_effect.insert(4, collector.NoDataCollected('a', 'b'))
self.collector_mock.retrieve.side_effect = side_effect
output = sorted(self.worker._do_collection(metrics,
timestamp_now).items(),
key=lambda x: x[1]['usage'])
self.assertEqual([
i for i in side_effect
if not isinstance(i, collector.NoDataCollected)
], output)
def check_time_state(timestamp=None, period=0, wait_periods=0):
"""Checks the state of a timestamp compared to the current time.
Returns the next timestamp based on the current timestamp and the period if
the next timestamp is inferior to the current time and the waiting period
or None if not.
:param timestamp: Current timestamp
:type timestamp: datetime.datetime
:param period: Period, in seconds
:type period: int
:param wait_periods: periods to wait before the current timestamp.
:type wait_periods: int
:rtype: datetime.datetime
"""
if not timestamp:
return tzutils.get_month_start()
period_delta = datetime.timedelta(seconds=period)
next_timestamp = tzutils.add_delta(timestamp, period_delta)
wait_time = wait_periods * period_delta
if tzutils.add_delta(next_timestamp, wait_time) < tzutils.localized_now():
return next_timestamp
return None
def initialize_data(self):
data = test_utils.generate_v2_storage_data(
start=tzutils.get_month_start(),
end=tzutils.localized_now().replace(hour=0),
)
self.storage.push([data])
def test_update_scope_processing_state_db(self):
timestamp = tzutils.localized_now()
self.worker.update_scope_processing_state_db(timestamp)
self.state_manager_set_state_mock.assert_has_calls(
[mock.call(self.worker._tenant_id, timestamp)])
def setUp(self):
self.local_now = tzutils.localized_now()
self.naive_now = utils.utcnow().replace(microsecond=0)
