Пример #1
0
    def show_aggregated_resource_rate(self, begin=None, end=None, tenant_ids=None, resource_id=None):
        model = models.AggregatedDataFrame

        # Boundary calculation
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        session = db.get_session()
        q = session.query(model.begin.label('begin'), model.end.label('end'), 
                          model.rate.label('rate'), model.qty.label('qty'), 
                          model.unit.label('unit'), model.unit_price.label('unit_price'),
                          model.begining_event_type.label('begin_event'), model.ending_event_type.label('end_event'))
        if tenant_ids:
            q = q.filter(
                model.tenant_id.in_(tenant_ids)
            )

        q = q.filter(
            model.begin >= begin,
            model.end <= end,
            model.res_type != '_NO_DATA_',
            model.resource_id == resource_id
            )
            
        resources_usage_rate = q.all()
        
        return [model.to_resource_usage_rate(entry) for entry in resources_usage_rate]
Пример #2
0
    def total(self,
              begin=None,
              end=None,
              tenant_id=None,
              service=None,
              all_tenants=False):
        """Return the amount to pay for a given period.

        """
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        if all_tenants:
            tenant_id = None
        else:
            tenant_context = pecan.request.context.tenant
            tenant_id = tenant_context if not tenant_id else tenant_id
        policy.authorize(pecan.request.context, 'report:get_total',
                         {"tenant_id": tenant_id})

        storage = pecan.request.storage_backend
        # FIXME(sheeprine): We should filter on user id.
        # Use keystone token information by default but make it overridable and
        # enforce it by policy engine
        total = storage.get_total(begin, end, tenant_id, service)

        # TODO(Aaron): `get_total` return a list of dict,
        # Get value of rate from index[0]
        total = total[0].get('rate', decimal.Decimal('0'))
        return total if total else decimal.Decimal('0')
Пример #3
0
    def summary(self,
                begin=None,
                end=None,
                tenant_id=None,
                service=None,
                groupby=None,
                all_tenants=False):
        """Return the summary to pay for a given period.

        """
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        if all_tenants:
            tenant_id = None
        else:
            tenant_context = pecan.request.context.tenant
            tenant_id = tenant_context if not tenant_id else tenant_id
        policy.authorize(pecan.request.context, 'report:get_summary',
                         {"tenant_id": tenant_id})
        storage = pecan.request.storage_backend

        summarymodels = []
        results = storage.get_total(begin,
                                    end,
                                    tenant_id,
                                    service,
                                    groupby=groupby)
        for result in results:
            summarymodel = report_models.SummaryModel(**result)
            summarymodels.append(summarymodel)

        return report_models.SummaryCollectionModel(summary=summarymodels)
Пример #4
0
    def get_total(self, begin=None, end=None, tenant_id=None, service=None, instance_id=None):
        model = models.RatedDataFrame

        # Boundary calculation
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        session = db.get_session()
        q = session.query(
            sqlalchemy.func.sum(model.rate).label('rate'))
        if tenant_id:
            q = q.filter(
                models.RatedDataFrame.tenant_id == tenant_id)
        if service:
            q = q.filter(
                models.RatedDataFrame.res_type == service)
        if instance_id:
            q = q.filter(
                models.RatedDataFrame.desc.like('%'+instance_id+'%'))

        q = q.filter(
            model.begin >= begin,
            model.end <= end)
        rate = q.scalar()
        return rate
Пример #5
0
    def get_total(self, begin=None, end=None, tenant_id=None, service=None, instance_id=None):
        model = models.RatedDataFrame

        # Boundary calculation
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        session = db.get_session()
        q = session.query(
            sqlalchemy.func.sum(model.rate).label('rate'))
        if tenant_id:
            q = q.filter(
                models.RatedDataFrame.tenant_id == tenant_id)
        if service:
            q = q.filter(
                models.RatedDataFrame.res_type == service)
        if instance_id:
            q = q.filter(
                models.RatedDataFrame.desc.like('%'+instance_id+'%'))

        q = q.filter(
            model.begin >= begin,
            model.end <= end)
        rate = q.scalar()
        return rate
Пример #6
0
    def get(self,
            groupby=None,
            filters={},
            begin=None,
            end=None,
            offset=0,
            limit=100):
        policy.authorize(flask.request.context, 'summary:get_summary',
                         {'tenant_id': flask.request.context.project_id})
        begin = begin or utils.get_month_start()
        end = end or utils.get_next_month()

        if not flask.request.context.is_admin:
            filters['project_id'] = flask.request.context.project_id

        total = self._storage.total(
            begin=begin,
            end=end,
            groupby=groupby,
            filters=filters,
            offset=offset,
            limit=limit,
            paginate=True,
        )
        columns = []
        if len(total['results']) > 0:
            columns = list(total['results'][0].keys())

        return {
            'total': total['total'],
            'columns': columns,
            'results': [list(res.values()) for res in total['results']]
        }
Пример #7
0
 def tenants_list(self):
     if not CONF.command.begin:
         CONF.command.begin = ck_utils.get_month_start()
     if not CONF.command.end:
         CONF.command.end = ck_utils.get_next_month()
     tenants = self._storage.get_tenants(CONF.command.begin,
                                         CONF.command.end)
     print('Tenant list:')
     for tenant in tenants:
         print(tenant)
Пример #8
0
    def check_state(self):
        timestamp = self._storage.get_state(self._tenant_id)
        if not timestamp:
            month_start = ck_utils.local2utc(ck_utils.get_month_start())
            return ck_utils.dt2ts(ck_utils.iso2dt(month_start))

        now = ck_utils.utcnow_ts()
        if timestamp + self._period + self._wait_time < now:
            return timestamp
        return 0
Пример #9
0
 def tenants_list(self):
     if not CONF.command.begin:
         CONF.command.begin = ck_utils.get_month_start()
     if not CONF.command.end:
         CONF.command.end = ck_utils.get_next_month()
     tenants = self._storage.get_tenants(CONF.command.begin,
                                         CONF.command.end)
     print('Tenant list:')
     for tenant in tenants:
         print(tenant)
Пример #10
0
    def check_state(self):
        timestamp = self._storage.get_state(self._tenant_id)
        if not timestamp:
            month_start = ck_utils.get_month_start()
            return ck_utils.dt2ts(month_start)

        now = ck_utils.utcnow_ts()
        next_timestamp = timestamp + self._period
        if next_timestamp + self._wait_time < now:
            return next_timestamp
        return 0
Пример #11
0
    def check_state(self):
        timestamp = self._storage.get_state(self._tenant_id)
        if not timestamp:
            month_start = ck_utils.get_month_start()
            return ck_utils.dt2ts(month_start)

        now = ck_utils.utcnow_ts()
        next_timestamp = timestamp + self._period
        if next_timestamp + self._wait_time < now:
            return next_timestamp
        return 0
Пример #12
0
    def _check_state(self, tenant_id):
        timestamp = self.storage.get_state(tenant_id)
        if not timestamp:
            month_start = ck_utils.get_month_start()
            return ck_utils.dt2ts(month_start)

        now = ck_utils.utcnow_ts()
        next_timestamp = timestamp + CONF.collect.period
        wait_time = CONF.collect.wait_periods * CONF.collect.period
        if next_timestamp + wait_time < now:
            return next_timestamp
        return 0
Пример #13
0
    def _check_state(self, tenant_id):
        timestamp = self.storage.get_state(tenant_id)
        if not timestamp:
            month_start = ck_utils.get_month_start()
            return ck_utils.dt2ts(month_start)

        now = ck_utils.utcnow_ts()
        next_timestamp = timestamp + CONF.collect.period
        wait_time = CONF.collect.wait_periods * CONF.collect.period
        if next_timestamp + wait_time < now:
            return next_timestamp
        return 0
Пример #14
0
    def tenants(self, begin=None, end=None):
        """Return the list of rated tenants.

        """
        policy.authorize(pecan.request.context, 'report:list_tenants', {})

        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        storage = pecan.request.storage_backend
        tenants = storage.get_tenants(begin, end)
        return tenants
Пример #15
0
    def get_all(self, begin=None, end=None, tenant_id=None,
                resource_type=None):
        """Return a list of rated resources for a time period and a tenant.

        :param begin: Start of the period
        :param end: End of the period
        :param tenant_id: UUID of the tenant to filter on.
        :param resource_type: Type of the resource to filter on.
        :return: Collection of DataFrame objects.
        """

        policy.authorize(pecan.request.context, 'storage:list_data_frames', {})

        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        begin_ts = ck_utils.dt2ts(begin)
        end_ts = ck_utils.dt2ts(end)
        backend = pecan.request.storage_backend
        dataframes = []
        try:
            frames = backend.get_time_frame(begin_ts,
                                            end_ts,
                                            tenant_id=tenant_id,
                                            res_type=resource_type)
            for frame in frames:
                for service, data_list in frame['usage'].items():
                    frame_tenant = None
                    resources = []
                    for data in data_list:
                        desc = data['desc'] if data['desc'] else {}
                        price = decimal.Decimal(str(data['rating']['price']))
                        resource = storage_models.RatedResource(
                            service=service,
                            desc=desc,
                            volume=data['vol']['qty'],
                            rating=price)
                        frame_tenant = data['tenant_id']
                        resources.append(resource)
                    dataframe = storage_models.DataFrame(
                        begin=ck_utils.iso2dt(frame['period']['begin']),
                        end=ck_utils.iso2dt(frame['period']['end']),
                        tenant_id=frame_tenant,
                        resources=resources)
                    dataframes.append(dataframe)
        except ck_storage.NoTimeFrame:
            pass
        return storage_models.DataFrameCollection(dataframes=dataframes)
Пример #16
0
    def get_tenants(self, begin=None, end=None):
        model = models.RatedDataFrame

        # Boundary calculation
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        session = db.get_session()
        q = utils.model_query(model, session).filter(model.begin >= begin,
                                                     model.end <= end)
        tenants = q.distinct().values(model.tenant_id)
        return [tenant.tenant_id for tenant in tenants]
Пример #17
0
    def summary(self,
                begin=None,
                end=None,
                tenant_id=None,
                service=None,
                groupby=None,
                all_tenants=False):
        """Return the summary to pay for a given period.

        """
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        if all_tenants:
            tenant_id = None
        else:
            tenant_context = pecan.request.context.project_id
            tenant_id = tenant_context if not tenant_id else tenant_id
        policy.authorize(pecan.request.context, 'report:get_summary',
                         {"tenant_id": tenant_id})
        storage = pecan.request.storage_backend

        scope_key = CONF.collect.scope_key
        storage_groupby = []
        if groupby is not None and 'tenant_id' in groupby:
            storage_groupby.append(scope_key)
        if groupby is not None and 'res_type' in groupby:
            storage_groupby.append('type')
        filters = {scope_key: tenant_id} if tenant_id else None
        result = storage.total(groupby=storage_groupby,
                               begin=begin,
                               end=end,
                               metric_types=service,
                               filters=filters)

        summarymodels = []
        for res in result['results']:
            kwargs = {
                'res_type': res.get('type') or res.get('res_type'),
                'tenant_id': res.get(scope_key) or res.get('tenant_id'),
                'begin': res['begin'],
                'end': res['end'],
                'rate': res['rate'],
            }
            summarymodel = report_models.SummaryModel(**kwargs)
            summarymodels.append(summarymodel)

        return report_models.SummaryCollectionModel(summary=summarymodels)
Пример #18
0
    def _check_begin_end(begin, end):
        if not begin:
            begin = utils.get_month_start()
        if not end:
            end = utils.get_next_month()
        if isinstance(begin, six.text_type):
            begin = utils.iso2dt(begin)
        if isinstance(begin, int):
            begin = utils.ts2dt(begin)
        if isinstance(end, six.text_type):
            end = utils.iso2dt(end)
        if isinstance(end, int):
            end = utils.ts2dt(end)

        return begin, end
Пример #19
0
    def _check_begin_end(begin, end):
        if not begin:
            begin = utils.get_month_start()
        if not end:
            end = utils.get_next_month()
        if isinstance(begin, six.text_type):
            begin = utils.iso2dt(begin)
        if isinstance(begin, int):
            begin = utils.ts2dt(begin)
        if isinstance(end, six.text_type):
            end = utils.iso2dt(end)
        if isinstance(end, int):
            end = utils.ts2dt(end)

        return begin, end
Пример #20
0
    def get_total(self,
                  begin=None,
                  end=None,
                  tenant_id=None,
                  service=None,
                  groupby=None):
        session = db.get_session()
        querymodels = [
            sqlalchemy.func.sum(self.frame_model.rate).label('rate')
        ]

        # Boundary calculation
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()
        if tenant_id:
            querymodels.append(self.frame_model.tenant_id)
        if service:
            querymodels.append(self.frame_model.res_type)
        if groupby:
            groupbyfields = groupby.split(",")
            for field in groupbyfields:
                field_obj = self.frame_model.__dict__.get(field, None)
                if field_obj and field_obj not in querymodels:
                    querymodels.append(field_obj)

        q = session.query(*querymodels)
        if tenant_id:
            q = q.filter(self.frame_model.tenant_id == tenant_id)
        if service:
            q = q.filter(self.frame_model.res_type == service)
        q = q.filter(self.frame_model.begin >= begin,
                     self.frame_model.end <= end,
                     self.frame_model.res_type != '_NO_DATA_')
        if groupby:
            q = q.group_by(groupby)

        # Order by sum(rate)
        q = q.order_by(sqlalchemy.func.sum(self.frame_model.rate))
        results = q.all()
        totallist = []
        for r in results:
            total = {model.name: value for model, value in zip(querymodels, r)}
            total["begin"] = begin
            total["end"] = end
            totallist.append(total)
        return totallist
Пример #21
0
    def get_tenants(self, begin=None, end=None):
        # Boundary calculation
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        session = db.get_session()
        q = utils.model_query(
            self.frame_model,
            session)
        q = q.filter(
            self.frame_model.begin >= begin,
            self.frame_model.end <= end)
        tenants = q.distinct().values(
            self.frame_model.tenant_id)
        return [tenant.tenant_id for tenant in tenants]
Пример #22
0
 def generate(self):
     if not CONF.command.tenant:
         if not CONF.command.begin:
             CONF.command.begin = ck_utils.get_month_start()
         if not CONF.command.end:
             CONF.command.end = ck_utils.get_next_month()
         tenants = self._storage.get_tenants(CONF.command.begin,
                                             CONF.command.end)
     else:
         tenants = [CONF.command.tenant]
     for tenant in tenants:
         wo = write_orchestrator.WriteOrchestrator(self._output, tenant,
                                                   self._storage,
                                                   CONF.output.basepath)
         wo.init_writing_pipeline()
         if not CONF.command.begin:
             wo.restart_month()
         wo.process()
Пример #23
0
 def generate(self):
     if not CONF.command.tenant:
         if not CONF.command.begin:
             CONF.command.begin = ck_utils.get_month_start()
         if not CONF.command.end:
             CONF.command.end = ck_utils.get_next_month()
         tenants = self._storage.get_tenants(CONF.command.begin,
                                             CONF.command.end)
     else:
         tenants = [CONF.command.tenant]
     for tenant in tenants:
         wo = write_orchestrator.WriteOrchestrator(self._output,
                                                   tenant,
                                                   self._storage,
                                                   CONF.output.basepath)
         wo.init_writing_pipeline()
         if not CONF.command.begin:
             wo.restart_month()
         wo.process()
Пример #24
0
 def get_time_frame(self, begin, end, **filters):
     if not begin:
         begin = ck_utils.get_month_start()
     if not end:
         end = ck_utils.get_next_month()
     session = db.get_session()
     q = utils.model_query(self.frame_model, session)
     q = q.filter(self.frame_model.begin >= begin,
                  self.frame_model.end <= end)
     for filter_name, filter_value in filters.items():
         if filter_value:
             q = q.filter(
                 getattr(self.frame_model, filter_name) == filter_value)
     if not filters.get('res_type'):
         q = q.filter(self.frame_model.res_type != '_NO_DATA_')
     count = q.count()
     if not count:
         raise NoTimeFrame()
     r = q.all()
     return [entry.to_cloudkitty(self._collector) for entry in r]
Пример #25
0
    def total(self,
              begin=None,
              end=None,
              tenant_id=None,
              service=None,
              all_tenants=False):
        """Return the amount to pay for a given period.

        """
        LOG.warning('/v1/report/total is deprecated, please use '
                    '/v1/report/summary instead.')
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        if all_tenants:
            tenant_id = None
        else:
            tenant_context = pecan.request.context.project_id
            tenant_id = tenant_context if not tenant_id else tenant_id
        policy.authorize(pecan.request.context, 'report:get_total',
                         {"tenant_id": tenant_id})

        storage = pecan.request.storage_backend
        # FIXME(sheeprine): We should filter on user id.
        # Use keystone token information by default but make it overridable and
        # enforce it by policy engine
        scope_key = CONF.collect.scope_key
        groupby = [scope_key]
        group_filters = {scope_key: tenant_id} if tenant_id else None
        total_resources = storage.total(groupby=groupby,
                                        begin=begin,
                                        end=end,
                                        metric_types=service,
                                        group_filters=group_filters)

        # TODO(Aaron): `get_total` return a list of dict,
        # Get value of rate from index[0]
        total = sum(total['rate'] for total in total_resources)
        return total if total else decimal.Decimal('0')
Пример #26
0
    def get_total(self, begin=None, end=None, tenant_id=None, service=None):
        # Boundary calculation
        if not begin:
            begin = ck_utils.get_month_start()
        if not end:
            end = ck_utils.get_next_month()

        session = db.get_session()
        q = session.query(
            sqlalchemy.func.sum(self.frame_model.rate).label('rate'))
        if tenant_id:
            q = q.filter(
                self.frame_model.tenant_id == tenant_id)
        if service:
            q = q.filter(
                self.frame_model.res_type == service)
        q = q.filter(
            self.frame_model.begin >= begin,
            self.frame_model.end <= end)
        rate = q.scalar()
        return rate
Пример #27
0
    def _get_metadata(self, resource_type, transformers):
        info = {}
        try:
            met = list(METRICS_CONF['metrics_units'][resource_type].values())
            info['unit'] = met[0]['unit']
        # NOTE(mc): deprecated second try kept for backward compatibility.
        except KeyError:
            LOG.warning('Error when trying to use yaml metrology conf.')
            LOG.warning('Fallback on the deprecated oslo config method.')
            try:
                info['unit'] = self.units_mappings[resource_type][1]
            except (KeyError, IndexError):
                info['unit'] = self.default_unit[1]

        start = ck_utils.dt2ts(ck_utils.get_month_start())
        end = ck_utils.dt2ts(ck_utils.get_month_end())

        try:
            resource = self.active_resources(resource_type, start, end,
                                             None)[0]
        except IndexError:
            resource = {}
        info['metadata'] = resource.get('dimensions', {}).keys()

        try:
            service_metrics = METRICS_CONF['services_metrics'][resource_type]
            for service_metric in service_metrics:
                metric, statistics = list(service_metric.items())[0]
                info['metadata'].append(metric)
        # NOTE(mc): deprecated second try kept for backward compatibility.
        except KeyError:
            LOG.warning('Error when trying to use yaml metrology conf.')
            LOG.warning('Fallback on the deprecated oslo config method.')
            try:
                for metric, statistics in self.metrics_mappings[resource_type]:
                    info['metadata'].append(metric)
            except (KeyError, IndexError):
                pass
        return info
Пример #28
0
    def _get_metadata(self, resource_type, transformers, conf):
        info = {}
        info['unit'] = conf['metrics'][resource_type]['unit']

        start = ck_utils.dt2ts(ck_utils.get_month_start())
        end = ck_utils.dt2ts(ck_utils.get_month_end())

        try:
            resource = self.active_resources(
                resource_type,
                start,
                end,
                None,
            )[0]
        except IndexError:
            resource = {}
        info['metadata'] = resource.get('dimensions', {}).keys()

        service_metrics = METRICS_CONF['services_metrics'][resource_type]
        for service_metric in service_metrics:
            metric, statistics = list(service_metric.items())[0]
            info['metadata'].append(metric)
        return info
Пример #29
0
 def current_month():
     month_start = ck_utils.get_month_start()
     return ck_utils.dt2ts(month_start)
Пример #30
0
 def last_month():
     month_start = ck_utils.get_month_start()
     month_end = ck_utils.get_month_end()
     start_ts = ck_utils.dt2ts(month_start)
     end_ts = ck_utils.dt2ts(month_end)
     return start_ts, end_ts
Пример #31
0
 def test_month_start_without_dt(self, patch_utcnow_mock):
     date = datetime.datetime(2014, 1, 1)
     trans_dt = ck_utils.get_month_start()
     self.assertEqual(date, trans_dt)
     patch_utcnow_mock.assert_called_once_with()
Пример #32
0
 def restart_month(self):
     self._load_state_manager_data()
     month_start = ck_utils.get_month_start()
     self.usage_end = ck_utils.dt2ts(month_start)
     self._update_state_manager_data()
Пример #33
0
 def restart_month(self):
     self._load_state_manager_data()
     month_start = ck_utils.get_month_start()
     self.usage_end = ck_utils.dt2ts(month_start)
     self._update_state_manager_data()
Пример #34
0
 def current_month():
     month_start = ck_utils.get_month_start()
     return ck_utils.dt2ts(month_start)
Пример #35
0
 def last_month():
     month_start = ck_utils.get_month_start()
     month_end = ck_utils.get_month_end()
     start_ts = ck_utils.dt2ts(month_start)
     end_ts = ck_utils.dt2ts(month_end)
     return start_ts, end_ts
Пример #36
0
 def _check_begin_end(begin, end):
     if not begin:
         begin = utils.get_month_start()
     if not end:
         end = utils.get_next_month()
     return begin, end
Пример #37
0
 def test_month_start_without_dt(self, patch_utcnow_mock):
     date = datetime.datetime(2014, 1, 1)
     trans_dt = ck_utils.get_month_start()
     self.assertEqual(date, trans_dt)
     patch_utcnow_mock.assert_called_once_with()
Пример #38
0
 def initialize_data(self):
     data = test_utils.generate_v2_storage_data(
         start=ck_utils.get_month_start(),
         end=ck_utils.utcnow().replace(hour=0),
     )
     self.storage.push([data])