def aggregate(self, report_id):
logbook.info("Get customer usage aggregation for {}", report_id)
customer = Customer.get_by_id(report_id.customer_id)
if not customer:
raise Exception("Customer %s not found" % report_id.customer_id)
with timed("get_usage simple"):
aggregated_usage = ServiceUsage.get_usage(customer, report_id.start, report_id.end)
tariffs = {}
services = set()
for usage in aggregated_usage:
service_id, tariff_id, cost, usage_volume = usage
services.add(service_id)
if not tariff_id:
logbook.error("ServiceUsage {} is not completed. Tariff is not filled", usage)
continue
tariff = Tariff.get_by_id(tariff_id)
tariff_report = tariffs.get(tariff_id)
if tariff_report is None:
tariff_report = self.tariff_report_type(tariff, customer)
tariffs[tariff_id] = tariff_report
tariff_report.add_usage(usage)
total = Counter()
for tariff_id, tariff in tariffs.items():
total_tariff, currency = tariff.aggregate()
total[currency] += total_tariff
for t, value in total.items():
total[t] = decimal_to_string(value)
logbook.info("Aggregated {} for {}. Services: {}", total, customer, services)
return self.prepare_result(list(tariffs.values()), total, customer, report_id.start, report_id.end)
def aggregate(self, report_id):
logbook.info("Get detailed customer usage aggregation for {}",
report_id)
customer = Customer.get_by_id(report_id.customer_id)
if not customer:
raise Exception("Customer %s not found" % report_id.customer_id)
with timed("get_usage simple"):
aggregated_usage = ServiceUsage.get_detailed_usage(
customer, report_id.start, report_id.end)
tariffs = {}
services = set()
for usage in aggregated_usage:
tariff = Tariff.get_by_id(usage.tariff_id)
tariff_report = tariffs.get(usage.tariff_id)
if tariff_report is None:
tariff_report = self.tariff_report_type(tariff, customer)
tariffs[usage.tariff_id] = tariff_report
tariff_report.add_usage(usage)
total = Counter()
for tariff_id, tariff in tariffs.items():
total_tariff, currency = tariff.aggregate()
total[currency] += total_tariff
for t, value in total.items():
total[t] = decimal_to_string(value)
logbook.info("Aggregated {} for {}. Services: {}", total, customer,
services)
return self.prepare_result(list(tariffs.values()), total, customer,
report_id.start, report_id.end)
def default_json(obj):
if isinstance(obj, Decimal):
return decimal_to_string(obj)
if isinstance(obj, datetime.datetime):
return {"$datetime$": obj.replace(microsecond=0).isoformat()}
if isinstance(obj, datetime.date):
return {"$date$": obj.isoformat()}
if hasattr(obj, "to_json"):
return obj.to_json()
raise TypeError("Cannot serialize %r" % obj)
def default_json(obj):
if isinstance(obj, Decimal):
return decimal_to_string(obj)
if isinstance(obj, datetime.datetime):
return {"$datetime$": obj.replace(microsecond=0).isoformat()}
if isinstance(obj, datetime.date):
return {"$date$": obj.isoformat()}
if hasattr(obj, "to_json"):
return obj.to_json()
raise TypeError("Cannot serialize %r" % obj)
def default(self, obj):
from utils.money import decimal_to_string
if isinstance(obj, datetime.datetime):
from pytz import utc
obj = obj.replace(microsecond=0, tzinfo=utc)
return obj.isoformat()
if isinstance(obj, datetime.date):
return obj.isoformat()
if isinstance(obj, Decimal):
return decimal_to_string(obj)
elif hasattr(obj, "to_json"):
return obj.to_json()
else:
return super(DateTimeJSONEncoder, self).default(obj)
def aggregate(self, report_id):
logbook.info("Get receipts aggregation for {}", report_id)
from model import AccountHistory, User
result = []
users = {user.user_id: user.name for user in User.query}
users[None] = ""
for row in AccountHistory.report(report_id.start, report_id.end):
r = {"email": row[0].email,
"date": row[1].date,
"amount": decimal_to_string(row[1].delta),
"currency": row[2].currency,
"comment": row[1].comment,
"user": users[row[1].user_id]}
result.append(r)
return result
def display(self, short=True):
service = Service.get_by_id(self.service_id)
return {"service": service.display(short), "price": decimal_to_string(self.price),
"need_changing": self.need_changing}
def test_collector(self):
start_time = datetime.datetime(2015, 3, 20, 9, 12)
end_time = datetime.datetime(2015, 5, 1)
project = Tenant("boss", start_time)
disk = Disk(project, "test_disk", start_time, 1234567890)
disk2 = Disk(project, "test_disk2", start_time, 3456)
volume = Volume(project, "figvam", datetime.datetime.now(), 2)
vm = Instance(project, "teamcity", "Nano")
vm2 = Instance(project, "teamcity1", "TestFlavor")
disk.repeat_message(start_time, end_time)
disk2.repeat_message(start_time, end_time)
vm.repeat_message(start_time, end_time)
vm2.repeat_message(start_time, end_time)
volume.repeat_message(start_time, end_time)
hour_price = Decimal(self.volume_size_price) * 2 + Decimal(self.image_size_price) * 3 + \
+ Decimal(self.nano_price)
with mock.patch(
"os_interfaces.openstack_wrapper.openstack") as openstack:
openstack.get_tenant_usage = project.usage
project.prepare_messages()
tenants_usage = self.collector.run_usage_collection(
end_time + datetime.timedelta(hours=10))
tenant_usage = next(iter(tenants_usage.values()))
self.assertTrue(tenant_usage)
self.assertEqual(self.collector.errors, 0)
for time_label, time_label_usage in tenant_usage.items():
withdraw = time_label_usage[1]
self.assertLess(withdraw - hour_price, 1e-6)
for usage in time_label_usage[0]:
if usage["service_id"] == "image.size":
self.assertEqual(usage.volume, 1234567890)
if usage['service_id'] == "volume.size":
self.assertEqual(usage.volume, 2 * 1073741824) # 2GB
self.assertEqual(
outbox[1].subject, '%s: Notice on service adding notification' %
conf.provider.cloud_name)
account = Customer.get_by_id(project.customer_id).account_dict()["RUB"]
self.assertEqual(account["balance"], Decimal(project.start_balance))
hours = int((end_time - start_time).total_seconds() // 3600) + 1
total_cost = hours * hour_price
self.assertLess(abs(account["withdraw"] - total_cost), 0.0001)
report = self.get_report(project.customer_id, start_time, end_time,
"json")
self.assertEqual(report["total"]["RUB"], decimal_to_string(total_cost))
report = self.get_report(project.customer_id,
start_time,
end_time,
"json",
report_is_ready=True)
report_ru = self.get_report(project.customer_id, start_time, end_time,
"csv")
customer = Customer.get_by_id(project.customer_id)
customer.locale = "en_us"
db.session.commit()
report_en = self.get_report(project.customer_id,
start_time,
end_time,
"csv",
filename="report_en.csv")
self.assertNotEqual(report_ru, report_en)
self.assertGreater(report_ru.count(b";"), 0)
self.assertEqual(report_en.count(b";"), 0)
# pdf en
self.get_report(project.customer_id, start_time, end_time, "pdf")
customer = Customer.get_by_id(project.customer_id)
customer.locale = "ru_ru"
db.session.commit()
# pdf ru
self.get_report(project.customer_id, start_time, end_time, "pdf")
# json after pdf
json_report = self.get_report(project.customer_id,
start_time,
end_time,
"json",
report_is_ready=True)
self.cabinet_client.auth(project.customer_email,
project.customer_password)
customer_report = self.get_report_me(start_time,
end_time,
"json",
report_is_ready=True)
self.assertEqual(json_report, customer_report)
self.get_report_me(start_time - datetime.timedelta(days=30),
start_time,
"csv",
filename="empty.csv")
self.get_report_me(start_time - datetime.timedelta(days=30),
start_time,
"pdf",
filename="empty.pdf")
def format_money(self, money, locale):
if isinstance(money, str) and self.money_pattern.match(money):
money = decimal_to_string(string_to_decimal(money), locale=locale)
return money
def test_collector(self):
start_time = datetime.datetime(2015, 3, 20, 9, 12)
end_time = datetime.datetime(2015, 5, 1)
project = Tenant("boss", start_time)
disk = Disk(project, "test_disk", start_time, 1234567890)
disk2 = Disk(project, "test_disk2", start_time, 3456)
volume = Volume(project, "figvam", datetime.datetime.now(), 2)
vm = Instance(project, "teamcity", "Nano")
vm2 = Instance(project, "teamcity1", "TestFlavor")
disk.repeat_message(start_time, end_time)
disk2.repeat_message(start_time, end_time)
vm.repeat_message(start_time, end_time)
vm2.repeat_message(start_time, end_time)
volume.repeat_message(start_time, end_time)
hour_price = Decimal(self.volume_size_price) * 2 + Decimal(self.image_size_price) * 3 + \
+ Decimal(self.nano_price)
with mock.patch("os_interfaces.openstack_wrapper.openstack") as openstack:
openstack.get_tenant_usage = project.usage
project.prepare_messages()
tenants_usage = self.collector.run_usage_collection(end_time + datetime.timedelta(hours=10))
tenant_usage = next(iter(tenants_usage.values()))
self.assertTrue(tenant_usage)
self.assertEqual(self.collector.errors, 0)
for time_label, time_label_usage in tenant_usage.items():
withdraw = time_label_usage[1]
self.assertLess(withdraw - hour_price, 1e-6)
for usage in time_label_usage[0]:
if usage["service_id"] == "image.size":
self.assertEqual(usage.volume, 1234567890)
if usage['service_id'] == "volume.size":
self.assertEqual(usage.volume, 2 * 1073741824) # 2GB
self.assertEqual(outbox[1].subject, '%s: Notice on service adding notification' % conf.provider.cloud_name)
account = Customer.get_by_id(project.customer_id).account_dict()["RUB"]
self.assertEqual(account["balance"], Decimal(project.start_balance))
hours = int((end_time - start_time).total_seconds() // 3600) + 1
total_cost = hours * hour_price
self.assertLess(abs(account["withdraw"] - total_cost), 0.0001)
report = self.get_report(project.customer_id, start_time, end_time, "json")
self.assertEqual(report["total"]["RUB"], decimal_to_string(total_cost))
report = self.get_report(project.customer_id, start_time, end_time, "json", report_is_ready=True)
report_ru = self.get_report(project.customer_id, start_time, end_time, "csv")
customer = Customer.get_by_id(project.customer_id)
customer.locale = "en_us"
db.session.commit()
report_en = self.get_report(project.customer_id, start_time, end_time, "csv", filename="report_en.csv")
self.assertNotEqual(report_ru, report_en)
self.assertGreater(report_ru.count(b";"), 0)
self.assertEqual(report_en.count(b";"), 0)
# pdf en
self.get_report(project.customer_id, start_time, end_time, "pdf")
customer = Customer.get_by_id(project.customer_id)
customer.locale = "ru_ru"
db.session.commit()
# pdf ru
self.get_report(project.customer_id, start_time, end_time, "pdf")
# json after pdf
json_report = self.get_report(project.customer_id, start_time, end_time, "json", report_is_ready=True)
self.cabinet_client.auth(project.customer_email, project.customer_password)
customer_report = self.get_report_me(start_time, end_time, "json", report_is_ready=True)
self.assertEqual(json_report, customer_report)
self.get_report_me(start_time - datetime.timedelta(days=30), start_time, "csv", filename="empty.csv")
self.get_report_me(start_time - datetime.timedelta(days=30), start_time, "pdf", filename="empty.pdf")