def configure_and_execute_overlap_test(self, schedule_get_all_mock, start_reprocess_time, end_reprocess_time): self.configure_schedules_mock(schedule_get_all_mock, start_reprocess_time, end_reprocess_time) scheduling_range = DateTimeRange( tzutils.utc_to_local(self.start_reprocess_time), tzutils.utc_to_local(self.end_reprocess_time)) scheduled_range = DateTimeRange( tzutils.local_to_utc(start_reprocess_time), tzutils.local_to_utc(end_reprocess_time)) expected_message = \ "400 Bad Request: Cannot schedule a reprocessing for scope " \ "[toStringMock] for reprocessing time [%s], because it already " \ "has a schedule for a similar time range [%s]." \ % (scheduling_range, scheduled_range) expected_message = re.escape(expected_message) self.assertRaisesRegex( http_exceptions.BadRequest, expected_message, self.endpoint.validate_reprocessing_schedules_overlaps, self.generate_all_scopes_object(), self.end_reprocess_time, self.start_reprocess_time) schedule_get_all_mock.assert_called_with( identifier=[self.scope_ids[0]])
def total(self, groupby=None, begin=None, end=None, metric_types=None, filters=None, offset=0, limit=1000, paginate=True): begin, end = self._check_begin_end(begin, end) total = self._conn.get_total(metric_types, begin, end, groupby, filters) output = [] for (series_name, series_groupby), points in total.items(): for point in points: # NOTE(peschk_l): InfluxDB returns all timestamps for a given # period and interval, even those with no data. This filters # out periods with no data if point['qty'] is not None and point['price'] is not None: output.append( self._get_total_elem(tzutils.utc_to_local(begin), tzutils.utc_to_local(end), groupby, series_groupby, point)) groupby = _sanitized_groupby(groupby) if groupby: output.sort(key=lambda x: [x[group] for group in groupby]) return { 'total': len(output), 'results': output[offset:offset + limit] if paginate else output, }
def test_get_total_groupby_tenant_and_restype(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) end = tzutils.utc_to_local(samples.SECOND_PERIOD_END) self.insert_data() total = self.storage.total( begin=begin, end=end, groupby=['project_id', 'type'])['results'] self.assertEqual(4, len(total)) self.assertEqual(0.1337, total[0]["rate"]) self.assertEqual(self._other_tenant_id, total[0]["tenant_id"]) self.assertEqual('image.size', total[0]["res_type"]) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"]) self.assertEqual(0.1337, total[1]["rate"]) self.assertEqual(self._tenant_id, total[1]["tenant_id"]) self.assertEqual('image.size', total[1]["res_type"]) self.assertEqual(begin, total[1]["begin"]) self.assertEqual(end, total[1]["end"]) self.assertEqual(0.84, total[2]["rate"]) self.assertEqual(self._other_tenant_id, total[2]["tenant_id"]) self.assertEqual('instance', total[2]["res_type"]) self.assertEqual(begin, total[2]["begin"]) self.assertEqual(end, total[2]["end"]) self.assertEqual(0.84, total[3]["rate"]) self.assertEqual(self._tenant_id, total[3]["tenant_id"]) self.assertEqual('instance', total[3]["res_type"]) self.assertEqual(begin, total[3]["begin"]) self.assertEqual(end, total[3]["end"])
def get_all(self, identifier=None, fetcher=None, collector=None, scope_key=None, active=1, limit=100, offset=0): """Returns the state of all scopes. This function returns the state of all scopes with support for optional filters. :param identifier: optional scope identifiers to filter on :type identifier: list :param fetcher: optional scope fetchers to filter on :type fetcher: list :param collector: optional collectors to filter on :type collector: list :param fetcher: optional fetchers to filter on :type fetcher: list :param scope_key: optional scope_keys to filter on :type scope_key: list :param active: optional active to filter scopes by status (active/deactivated) :type active: int :param limit: optional to restrict the projection :type limit: int :param offset: optional to shift the projection :type offset: int """ session = db.get_session() session.begin() q = utils.model_query(self.model, session) if identifier: q = q.filter( self.model.identifier.in_(to_list_if_needed(identifier))) if fetcher: q = q.filter(self.model.fetcher.in_(to_list_if_needed(fetcher))) if collector: q = q.filter(self.model.collector.in_( to_list_if_needed(collector))) if scope_key: q = q.filter(self.model.scope_key.in_( to_list_if_needed(scope_key))) if active is not None and active != []: q = q.filter(self.model.active.in_(to_list_if_needed(active))) q = apply_offset_and_limit(limit, offset, q) r = q.all() session.close() for item in r: item.last_processed_timestamp = tzutils.utc_to_local( item.last_processed_timestamp) item.scope_activation_toggle_date = tzutils.utc_to_local( item.scope_activation_toggle_date) return r
def _localize_total(iterable): for elem in iterable: begin = elem['begin'] end = elem['end'] if begin: elem['begin'] = tzutils.utc_to_local(begin) if end: elem['end'] = tzutils.utc_to_local(end)
def test_get_total_filtering_on_one_period(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) end = tzutils.utc_to_local(samples.FIRST_PERIOD_END) self.insert_data() total = self.storage.total(begin=begin, end=end)['results'] self.assertEqual(1, len(total)) self.assertEqual(1.1074, total[0]["rate"]) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"])
def test_get_total_without_filter_but_timestamp(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) end = tzutils.utc_to_local(samples.SECOND_PERIOD_END) self.insert_data() total = self.storage.total(begin=begin, end=end)['results'] # FIXME(sheeprine): floating point error (transition to decimal) self.assertEqual(1, len(total)) self.assertEqual(1.9473999999999998, total[0]["rate"]) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"])
def test_get_empty_total(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN - datetime.timedelta(seconds=3600)) end = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) self.insert_data() total = self.storage.total(begin=begin, end=end)['results'] self.assertEqual(1, len(total)) self.assertEqual(total[0]["rate"], 0) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"])
def test_get_total_filtering_on_service(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) end = tzutils.utc_to_local(samples.FIRST_PERIOD_END) self.insert_data() total = self.storage.total(begin=begin, end=end, metric_types='instance')['results'] self.assertEqual(1, len(total)) self.assertEqual(0.84, total[0]["rate"]) self.assertEqual('instance', total[0]["res_type"]) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"])
def test_get_total_filtering_on_one_period_and_one_tenant(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) end = tzutils.utc_to_local(samples.FIRST_PERIOD_END) self.insert_data() filters = {'project_id': self._tenant_id} total = self.storage.total(begin=begin, end=end, filters=filters)['results'] self.assertEqual(1, len(total)) self.assertEqual(0.5537, total[0]["rate"]) self.assertEqual(self._tenant_id, total[0]["tenant_id"]) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"])
def test_get_total_groupby_restype(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) end = tzutils.utc_to_local(samples.SECOND_PERIOD_END) self.insert_data() total = self.storage.total(begin=begin, end=end, groupby=['type'])['results'] self.assertEqual(2, len(total)) self.assertEqual(0.2674, total[0]["rate"]) self.assertEqual('image.size', total[0]["res_type"]) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"]) self.assertEqual(1.68, total[1]["rate"]) self.assertEqual('instance', total[1]["res_type"]) self.assertEqual(begin, total[1]["begin"]) self.assertEqual(end, total[1]["end"])
def test_get_total_groupby_tenant(self): begin = tzutils.utc_to_local(samples.FIRST_PERIOD_BEGIN) end = tzutils.utc_to_local(samples.SECOND_PERIOD_END) self.insert_data() total = self.storage.total(begin=begin, end=end, groupby=['project_id'])['results'] self.assertEqual(2, len(total)) self.assertEqual(0.9737, total[0]["rate"]) self.assertEqual(self._other_tenant_id, total[0]["tenant_id"]) self.assertEqual(begin, total[0]["begin"]) self.assertEqual(end, total[0]["end"]) self.assertEqual(0.9737, total[1]["rate"]) self.assertEqual(self._tenant_id, total[1]["tenant_id"]) self.assertEqual(begin, total[1]["begin"]) self.assertEqual(end, total[1]["end"])
def total(self, groupby=None, begin=None, end=None, metric_types=None, filters=None, offset=0, limit=1000, paginate=True, custom_fields="SUM(qty) AS qty, SUM(price) AS rate"): begin, end = self._check_begin_end(begin, end) total = self._conn.get_total(metric_types, begin, end, custom_fields, groupby, filters) output = [] for (series_name, series_groupby), points in total.items(): for point in points: # NOTE(peschk_l): InfluxDB returns all timestamps for a given # period and interval, even those with no data. This filters # out periods with no data # NOTE (rafaelweingartner): the summary get API is allowing # users to customize the report. Therefore, we only ignore # data points, if all of the entries have None values. # Otherwise, they are presented to the user. if [k for k in point.keys() if point[k]]: output.append( self._get_total_elem(tzutils.utc_to_local(begin), tzutils.utc_to_local(end), groupby, series_groupby, point)) groupby = _sanitized_groupby(groupby) if groupby: output.sort(key=lambda x: [x[group] for group in groupby]) return { 'total': len(output), 'results': output[offset:offset + limit] if paginate else output, }
def init_data(self): project_ids = [self._project_id, self._other_project_id] start_base = tzutils.utc_to_local(datetime.datetime(2018, 1, 1)) for i in range(3): start_delta = datetime.timedelta(seconds=3600 * i) end_delta = start_delta + datetime.timedelta(seconds=3600) start = tzutils.add_delta(start_base, start_delta) end = tzutils.add_delta(start_base, end_delta) data = test_utils.generate_v2_storage_data(project_ids=project_ids, start=start, end=end) self.data.append(data) self.storage.push([data])
def get_all(self, identifier=None, fetcher=None, collector=None, scope_key=None, limit=100, offset=0): """Returns the state of all scopes. This function returns the state of all scopes with support for optional filters. :param identifier: optional scope identifiers to filter on :type identifier: list :param fetcher: optional scope fetchers to filter on :type fetcher: list :param collector: optional collectors to filter on :type collector: list :param fetcher: optional fetchers to filter on :type fetcher: list :param scope_key: optional scope_keys to filter on :type scope_key: list """ session = db.get_session() session.begin() q = utils.model_query(self.model, session) if identifier: q = q.filter(self.model.identifier.in_(identifier)) if fetcher: q = q.filter(self.model.fetcher.in_(fetcher)) if collector: q = q.filter(self.model.collector.in_(collector)) if scope_key: q = q.filter(self.model.scope_key.in_(scope_key)) q = q.offset(offset).limit(limit) r = q.all() session.close() for item in r: item.state = tzutils.utc_to_local(item.state) return r
def get_state(self, identifier, fetcher=None, collector=None, scope_key=None): """Get the state of a scope. :param identifier: Identifier of the scope :type identifier: str :param fetcher: Fetcher associated to the scope :type fetcher: str :param collector: Collector associated to the scope :type collector: str :param scope_key: scope_key associated to the scope :type scope_key: str :rtype: datetime.datetime """ session = db.get_session() session.begin() r = self._get_db_item( session, identifier, fetcher, collector, scope_key) session.close() return tzutils.utc_to_local(r.state) if r else None
def _normalize_time(t): if isinstance(t, datetime.datetime): return tzutils.utc_to_local(t) return tzutils.dt_from_iso(t)
def test_utc_to_local(self): self.assertEqual(tzutils.utc_to_local(self.naive_now), self.local_now)
def test_reset_state(self): coord_start_patch = mock.patch.object( coordination.CoordinationDriverWithExecutor, 'start') lock_acquire_patch = mock.patch.object(file.FileLock, 'acquire', return_value=True) storage_delete_patch = mock.patch.object(influx.InfluxStorage, 'delete') state_set_patch = mock.patch.object(storage_state.StateManager, 'set_last_processed_timestamp') with coord_start_patch, lock_acquire_patch, \ storage_delete_patch as sd, state_set_patch as ss: endpoint = orchestrator.ScopeEndpoint() endpoint.reset_state({}, { 'scopes': [ { 'scope_id': 'f266f30b11f246b589fd266f85eeec39', 'scope_key': 'project_id', 'collector': 'prometheus', 'fetcher': 'prometheus', }, { 'scope_id': '4dfb25b0947c4f5481daf7b948c14187', 'scope_key': 'project_id', 'collector': 'gnocchi', 'fetcher': 'gnocchi', }, ], 'last_processed_timestamp': '20190716T085501Z', }) sd.assert_has_calls([ mock.call(begin=tzutils.utc_to_local( datetime.datetime(2019, 7, 16, 8, 55, 1)), end=None, filters={ 'project_id': 'f266f30b11f246b589fd266f85eeec39' }), mock.call( begin=tzutils.utc_to_local( datetime.datetime(2019, 7, 16, 8, 55, 1)), end=None, filters={'project_id': '4dfb25b0947c4f5481daf7b948c14187'}, ) ], any_order=True) ss.assert_has_calls([ mock.call('f266f30b11f246b589fd266f85eeec39', tzutils.utc_to_local( datetime.datetime(2019, 7, 16, 8, 55, 1)), scope_key='project_id', collector='prometheus', fetcher='prometheus'), mock.call('4dfb25b0947c4f5481daf7b948c14187', tzutils.utc_to_local( datetime.datetime(2019, 7, 16, 8, 55, 1)), scope_key='project_id', collector='gnocchi', fetcher='gnocchi') ], any_order=True)