def get_withdraw(cls, customer, start, finish):
start_tl = TimeLabel(start).label
finish_tl = TimeLabel(finish).label
query = db.session.query(cls.currency, func.sum(cls.cost)).\
filter(cls.tenant_id == customer.os_tenant_id, cls.time_label >= start_tl, cls.time_label < finish_tl)
query = query.group_by(cls.currency)
return dict(query)
def get_usage(cls, customer, start, finish):
tenant_id = customer.os_tenant_id
start_tl = TimeLabel(start).label
finish_tl = TimeLabel(finish).label
query = db.session.query(cls.service_id, cls.tariff_id, func.sum(cls.cost), func.sum(cls.usage_volume)).\
filter(cls.tenant_id == tenant_id, cls.time_label >= start_tl, cls.time_label < finish_tl)
query = query.group_by(cls.tariff_id, cls.service_id)
return query
def get_detailed_usage(cls, customer, start, finish):
tenant_id = customer.os_tenant_id
start_tl = TimeLabel(start).label
finish_tl = TimeLabel(finish).label
query = cls.query.filter(cls.tenant_id == tenant_id,
cls.time_label >= start_tl,
cls.time_label < finish_tl)
return query
def test_time_label_day(self):
start = TimeLabel(datetime.datetime(2013, 2, 3, 4, 5, 6))
finish = TimeLabel(datetime.datetime(2013, 2, 3, 4, 5, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 3, 4, 6, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 3, 4, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 3, 5, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 4, 5, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203", "20130204"])
finish = TimeLabel(datetime.datetime(2013, 2, 5, 5, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203", "20130204", "20130205"])
finish = TimeLabel(datetime.datetime(2013, 2, 5, 0, 0, 1))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203", "20130204", "20130205"])
finish = TimeLabel(datetime.datetime(2013, 3, 5, 0, 0, 1))
self.assertEqual(list(TimeLabel.days(start, finish)),
["20130203", "20130204", "20130205", "20130206", "20130207",
"20130208", "20130209", "20130210", "20130211", "20130212",
"20130213", "20130214", "20130215", "20130216", "20130217",
"20130218", "20130219", "20130220", "20130221", "20130222",
"20130223", "20130224", "20130225", "20130226", "20130227",
"20130228", "20130301", "20130302", "20130303", "20130304", "20130305"])
def test_utilization_time_label(self):
self.assertEqual(
TimeLabel(10 * 3600 + 30 * 60 + 1000 * 3600 * 24).label,
"1972092710")
self.assertEqual(
TimeLabel(00 * 3600 + 00 * 60 + 1000 * 3600 * 24).label,
"1972092700")
start = 2 * 3600 + 1000 * 3600 * 24
for x in range(start, start + 3600):
self.assertEqual(TimeLabel(x).label, "1972092702")
self.assertEqual(
TimeLabel(datetime.datetime(2013, 2, 3, 4, 5, 6)).label,
"2013020304")
self.assertEqual(
TimeLabel(datetime.date(2013, 2, 3)).label, "2013020300")
t1 = TimeLabel.from_str("2013020311")
t2 = TimeLabel.from_str("2013020312")
t3 = TimeLabel.from_str("2013020315")
self.assertEqual(t1 - t1, 0)
self.assertEqual(t2 - t1, 1)
self.assertEqual(t3 - t1, 4)
self.assertEqual(t3 - t2, 3)
def test_default_to_from_volume_case(self):
"""
Unless all image refs contain something, assume booted from volume.
"""
data = [{
'timestamp': testdata.t0,
'metadata': {
'image_ref': ""
}
}, {
'timestamp': testdata.t0_30,
'metadata': {
'image_ref': "d5a4f118023928195f4ef"
}
}, {
'timestamp': testdata.t1,
'metadata': {
'image_ref': "None"
}
}]
xform = transformers.FromImage()
usage = xform.transform_usage(
'instance', [OldSample(None, event) for event in data],
TimeLabel(testdata.t0))
self.assertEqual([], usage)
def test_basic_sum(self):
"""
Tests that the transformer correctly calculate the sum value.
"""
data = [
{
'timestamp': p('2015-01-01T00:00:00'),
'counter_volume': 1
},
{
'timestamp': p('2015-01-01T00:10:00'),
'counter_volume': 1
},
{
'timestamp': p('2015-01-01T01:00:00'),
'counter_volume': 1
},
]
xform = transformers.GaugeSum()
usage = xform.transform_usage('fake_meter', self.make_sample(data),
TimeLabel(testdata.t0))
self.assertEqual(usage[0].volume, 2)
def test_all_same_values(self):
"""
Tests that that transformer correctly grabs any value,
when all values are the same.
"""
data = [
{
'timestamp': testdata.t0,
'counter_volume': 25
},
{
'timestamp': testdata.t0_30,
'counter_volume': 25
},
{
'timestamp': testdata.t1,
'counter_volume': 25
},
]
xform = transformers.GaugeMax()
usage = xform.transform_usage('some_meter', self.make_sample(data),
TimeLabel(testdata.t0))
self.assertEqual(usage[0].volume, 25)
def test_basic_sum(self):
"""Tests that the transformer correctly calculate the sum value.
"""
data = [
{
'timestamp': p('2015-01-01T00:00:00'),
'counter_volume': 1
},
{
'timestamp': p('2015-01-01T00:10:00'),
'counter_volume': 0
},
{
'timestamp': p('2015-01-01T01:00:00'),
'counter_volume': 2
},
]
xform = transformers.GaugeNetworkService()
usage = xform.transform_usage('fake_meter', self.make_sample(data),
TimeLabel(testdata.t0))
self.assertEqual(usage[0].end - usage[0].start,
datetime.timedelta(hours=1, seconds=-1))
def test_from_image_case_highest_size(self):
"""
If all image refs contain something,
should return entry with highest size from data.
"""
data = [{
'timestamp': testdata.t0,
'metadata': {
'image_ref': "d5a4f118023928195f4ef",
'root_gb': "20"
}
}, {
'timestamp': testdata.t0_30,
'metadata': {
'image_ref': "d5a4f118023928195f4ef",
'root_gb': "60"
}
}, {
'timestamp': testdata.t1,
'metadata': {
'image_ref': "d5a4f118023928195f4ef",
'root_gb': "20"
}
}]
xform = transformers.FromImage()
usage = xform.transform_usage('instance', self.make_sample(data),
TimeLabel(testdata.t0))
self.assertEqual(usage[0].end - usage[0].start,
datetime.timedelta(hours=1, seconds=-1))
def customers_get_usage(cls, start, finish):
from model import Customer
start_tl = TimeLabel(start).label
finish_tl = TimeLabel(finish).label
deleted_gap = timedelta(seconds=conf.report.deleted_gap)
active_customers = Customer.query.filter((Customer.deleted == None) | (
Customer.deleted < start - deleted_gap))
result = {}
for customer in active_customers:
if customer.os_tenant_id:
query = db.session.query(cls.currency, func.sum(cls.cost)).\
filter(cls.tenant_id == customer.os_tenant_id,
cls.time_label >= start_tl, cls.time_label < finish_tl)
query = query.group_by(cls.currency)
result[customer] = query.all()
return result
def test_utilization_time_label(self):
self.assertEqual(TimeLabel(10 * 3600 + 30 * 60 + 1000 * 3600 * 24).label, "1972092710")
self.assertEqual(TimeLabel(00 * 3600 + 00 * 60 + 1000 * 3600 * 24).label, "1972092700")
start = 2 * 3600 + 1000 * 3600 * 24
for x in range(start, start + 3600):
self.assertEqual(TimeLabel(x).label, "1972092702")
self.assertEqual(TimeLabel(datetime.datetime(2013, 2, 3, 4, 5, 6)).label, "2013020304")
self.assertEqual(TimeLabel(datetime.date(2013, 2, 3)).label, "2013020300")
t1 = TimeLabel.from_str("2013020311")
t2 = TimeLabel.from_str("2013020312")
t3 = TimeLabel.from_str("2013020315")
self.assertEqual(t1 - t1, 0)
self.assertEqual(t2 - t1, 1)
self.assertEqual(t3 - t1, 4)
self.assertEqual(t3 - t2, 3)
def collect_usage(self, tenant, mutex, end=None):
# Collects usage for a given tenant from when they were last collected,
# up to the given end, and breaks the range into one hour windows.
end = end or datetime.utcnow()
time_label = TimeLabel(tenant.last_collected + timedelta(minutes=1))
end_time_label = TimeLabel(end)
usage = {}
logbook.info(
'collect_usage for {}, from {} till {} (last_collected: {})',
tenant, time_label, end_time_label, tenant.last_collected)
customer = Customer.get_by_tenant_id(tenant.tenant_id)
if not customer:
logbook.error("Customer for tenant {} not found", tenant)
return usage
while time_label < end_time_label:
try:
usages = self._collect_usage(tenant, time_label, customer)
tenant.last_collected = time_label.datetime_range()[1]
if usages:
db.session.add(customer)
total_cost = customer.calculate_usage_cost(usages)
customer.withdraw(total_cost)
if not conf.test:
db.session.commit()
usage[time_label] = [usage.to_dict()
for usage in usages], total_cost
except Exception:
self.errors += 1
import traceback
traceback.print_exc()
logbook.exception("Usage process failed for {} and {}", tenant,
time_label)
db.session.rollback()
return usage
time_label = time_label.next()
mutex.update_ttl()
return usage
def fake_usage(customer, start, finish, service_id, resource_id, volume, resource_name=None):
from model import ServiceUsage
from fitter.aggregation.timelabel import TimeLabel
if customer.os_tenant_id is None:
raise errors.TenantIsnotCreated()
time_label = TimeLabel(start)
finish_time_label = TimeLabel(finish)
total_cost = Decimal(0)
while time_label <= finish_time_label:
st, fn = time_label.datetime_range()
st = max(st, start)
fn = min(fn, finish)
service_usage = ServiceUsage(customer.os_tenant_id, service_id, time_label, resource_id,
customer.tariff, volume, st, fn, resource_name=resource_name)
db.session.add(service_usage)
cost = customer.calculate_usage_cost([service_usage])
customer.withdraw(cost)
total_cost += cost
time_label = time_label.next()
return total_cost
def collect_usage(self, tenant, mutex, end=None):
# Collects usage for a given tenant from when they were last collected,
# up to the given end, and breaks the range into one hour windows.
end = end or datetime.utcnow()
time_label = TimeLabel(tenant.last_collected + timedelta(minutes=1))
end_time_label = TimeLabel(end)
usage = {}
logbook.info('collect_usage for {}, from {} till {} (last_collected: {})',
tenant, time_label, end_time_label, tenant.last_collected)
customer = Customer.get_by_tenant_id(tenant.tenant_id)
if not customer:
logbook.error("Customer for tenant {} not found", tenant)
return usage
while time_label < end_time_label:
try:
usages = self._collect_usage(tenant, time_label, customer)
tenant.last_collected = time_label.datetime_range()[1]
if usages:
db.session.add(customer)
total_cost = customer.calculate_usage_cost(usages)
customer.withdraw(total_cost)
if not conf.test:
db.session.commit()
usage[time_label] = [usage.to_dict() for usage in usages], total_cost
except Exception:
self.errors += 1
import traceback
traceback.print_exc()
logbook.exception("Usage process failed for {} and {}", tenant, time_label)
db.session.rollback()
return usage
time_label = time_label.next()
mutex.update_ttl()
return usage
def test_from_volume_case(self):
"""
If instance is booted from volume transformer should return none.
"""
data = [{
'timestamp': testdata.t0,
'metadata': {
'image_ref': ""
}
}, {
'timestamp': testdata.t0_30,
'metadata': {
'image_ref': "None"
}
}, {
'timestamp': testdata.t1,
'metadata': {
'image_ref': "None"
}
}]
data2 = [{
'timestamp': testdata.t0_30,
'metadata': {
'image_ref': "None"
}
}]
xform = transformers.FromImage()
usage = xform.transform_usage('instance', self.make_sample(data),
TimeLabel(testdata.t0))
usage2 = xform.transform_usage('instance', self.make_sample(data2),
TimeLabel(testdata.t0))
self.assertEqual([], usage)
self.assertEqual([], usage2)
def test_all_different_values(self):
"""
Tests that the transformer correctly grabs the highest value,
when all values are different.
"""
data = [
{
'timestamp': testdata.t0,
'counter_volume': 12
},
{
'timestamp': testdata.t0_10,
'counter_volume': 3
},
{
'timestamp': testdata.t0_20,
'counter_volume': 7
},
{
'timestamp': testdata.t0_30,
'counter_volume': 3
},
{
'timestamp': testdata.t0_40,
'counter_volume': 25
},
{
'timestamp': testdata.t0_50,
'counter_volume': 2
},
{
'timestamp': testdata.t1,
'counter_volume': 6
},
]
xform = transformers.GaugeMax()
usage = xform.transform_usage('some_meter', self.make_sample(data),
TimeLabel(testdata.t0))
self.assertEqual(usage[0].volume, 25)
def add_usage(self, usage):
time_label = TimeLabel.from_str(usage.time_label)
current_data = self.time_labels.get(time_label.label)
if current_data:
logbook.error("At least two records for resource {} and time_label: {}. "
"Usages will be summarized. Usage: {}",
self.resource_id, time_label, usage)
return
measure_type = self.service.measure.measure_type if self.service else Measure.QUANTITATIVE
quantitative_service = measure_type == Measure.QUANTITATIVE
resource_usage_class = ResourceUsageQuantity if quantitative_service else ResourceUsageTime
resource_usage = resource_usage_class(time_label)
self.time_labels[time_label.label] = resource_usage
resource_usage.add_usage(time_label, usage)
if not quantitative_service:
self.merge_with_neighbor(time_label, resource_usage)
self.total_cost += usage.cost or Decimal(0)
self.total_usage_volume += usage.usage_volume or 0
def add_usage(self, usage):
time_label = TimeLabel.from_str(usage.time_label)
current_data = self.time_labels.get(time_label.label)
if current_data:
logbook.error(
"At least two records for resource {} and time_label: {}. "
"Usages will be summarized. Usage: {}", self.resource_id,
time_label, usage)
return
measure_type = self.service.measure.measure_type if self.service else Measure.QUANTITATIVE
quantitative_service = measure_type == Measure.QUANTITATIVE
resource_usage_class = ResourceUsageQuantity if quantitative_service else ResourceUsageTime
resource_usage = resource_usage_class(time_label)
self.time_labels[time_label.label] = resource_usage
resource_usage.add_usage(time_label, usage)
if not quantitative_service:
self.merge_with_neighbor(time_label, resource_usage)
self.total_cost += usage.cost or Decimal(0)
self.total_usage_volume += usage.usage_volume or 0
def test_time_label_day(self):
start = TimeLabel(datetime.datetime(2013, 2, 3, 4, 5, 6))
finish = TimeLabel(datetime.datetime(2013, 2, 3, 4, 5, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 3, 4, 6, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 3, 4, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 3, 5, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)), ["20130203"])
finish = TimeLabel(datetime.datetime(2013, 2, 4, 5, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)),
["20130203", "20130204"])
finish = TimeLabel(datetime.datetime(2013, 2, 5, 5, 16, 7))
self.assertEqual(list(TimeLabel.days(start, finish)),
["20130203", "20130204", "20130205"])
finish = TimeLabel(datetime.datetime(2013, 2, 5, 0, 0, 1))
self.assertEqual(list(TimeLabel.days(start, finish)),
["20130203", "20130204", "20130205"])
finish = TimeLabel(datetime.datetime(2013, 3, 5, 0, 0, 1))
self.assertEqual(list(TimeLabel.days(start, finish)), [
"20130203", "20130204", "20130205", "20130206", "20130207",
"20130208", "20130209", "20130210", "20130211", "20130212",
"20130213", "20130214", "20130215", "20130216", "20130217",
"20130218", "20130219", "20130220", "20130221", "20130222",
"20130223", "20130224", "20130225", "20130226", "20130227",
"20130228", "20130301", "20130302", "20130303", "20130304",
"20130305"
])
def _run_transform(self, data):
xform = transformers.Uptime()
return xform.transform_usage('state', self.make_sample(data),
TimeLabel(testdata.t0))
def test_max_volume(self):
"""
Test empty volume value
"""
data = [
{
'timestamp': testdata.t0,
'counter_volume': None
},
{
'timestamp': testdata.t0_10,
'counter_volume': None
},
{
'timestamp': testdata.t0_20,
'counter_volume': None
},
{
'timestamp': testdata.t0_30,
'counter_volume': None
},
{
'timestamp': testdata.t0_40,
'counter_volume': None
},
{
'timestamp': testdata.t0_50,
'counter_volume': None
},
{
'timestamp': testdata.t1,
'counter_volume': None
},
]
xform = transformers.GaugeMax()
usage = xform.transform_usage('some_meter', self.make_sample(data),
TimeLabel(testdata.t0))
self.assertEqual(usage[0].volume, 0)
data = [
{
'timestamp': testdata.t0,
'volume': None
},
{
'timestamp': testdata.t0_10,
'volume': None
},
{
'timestamp': testdata.t0_20,
'volume': None
},
{
'timestamp': testdata.t0_30,
'volume': None
},
{
'timestamp': testdata.t0_40,
'volume': None
},
{
'timestamp': testdata.t0_50,
'volume': None
},
{
'timestamp': testdata.t1,
'volume': None
},
]
xform = transformers.GaugeMax()
usage = xform.transform_usage('some_meter', self.make_sample(data),
TimeLabel(testdata.t0))
self.assertEqual(usage[0].volume, 0)
