def migrate_images(glance_url):
glance_client = client.RestClient()
glance_client.auth_headers = {"x-auth-token": global_conf.admin_token}
glance_client.management_url = glance_url
tariffs = db_api.tariff_map()
accounts = {}
images = json.loads(glance_client.get("/images/detail"))["images"]
for project_id in (img1["owner"] for img1 in images if img1["owner"]):
accounts[project_id] = \
db_api.account_get_or_create(project_id).id
for img1 in images:
if not img1["owner"]:
continue
account_id = accounts[img1["owner"]]
img2 = db_api.resource_get_or_create(
account_id, None,
ResourceTypes.Image,
img1["id"]
)
seg = Segment(
resource_id=img2.id,
cost=img1["size"] * tariffs.get(ResourceTypes.Image, 1) / (1024.0 ** 3),
begin_at=utils.str_to_datetime(img1["created_at"]),
end_at=utils.str_to_datetime(img1["deleted_at"]))
db.session.add(seg)
db.session.commit()
def migrate_images(glance_url):
glance_client = client.RestClient()
glance_client.auth_headers = {"x-auth-token": global_conf.admin_token}
glance_client.management_url = glance_url
tariffs = db_api.tariff_map()
accounts = {}
images = json.loads(glance_client.get("/images/detail"))["images"]
for project_id in (img1["owner"] for img1 in images if img1["owner"]):
accounts[project_id] = \
db_api.account_get_or_create(project_id).id
for img1 in images:
if not img1["owner"]:
continue
account_id = accounts[img1["owner"]]
img2 = db_api.resource_get_or_create(account_id, None,
ResourceTypes.Image, img1["id"])
seg = Segment(resource_id=img2.id,
cost=img1["size"] * tariffs.get(ResourceTypes.Image, 1) /
(1024.0**3),
begin_at=utils.str_to_datetime(img1["created_at"]),
end_at=utils.str_to_datetime(img1["deleted_at"]))
db.session.add(seg)
db.session.commit()
def migrate_instances(old_db_url):
engine1 = create_engine(old_db_url)
tariffs = db_api.tariff_map()
instance_resources = ("local_gb", "memory_mb", "vcpus")
instance_info_attrs = ("id", "instance_id", "project_id", "local_gb",
"memory_mb", "vcpus")
instance_segment_attrs = ("id", "instance_info_id", "segment_type",
"begin_at", "end_at")
instance_infos = {}
accounts = {}
for inst1 in engine1.execute(
"select distinct project_id from billing_instance_info"):
accounts[inst1.project_id] = \
db_api.account_get_or_create(inst1.project_id).id
for inst1 in engine1.execute("select %s from billing_instance_info" %
", ".join(instance_info_attrs)):
account_id = accounts[inst1.project_id]
inst2 = db_api.resource_get_or_create(account_id, None,
ResourceTypes.Instance,
inst1.instance_id)
inst_dict = {
"inst1": inst1,
"inst2": inst2,
}
for rtype in instance_resources:
inst_dict[rtype + "_id"] = db_api.resource_get_or_create(
account_id, inst2.id, rtype, None)
instance_infos[inst1.id] = inst_dict
for iseg in engine1.execute("select %s from billing_instance_segment" %
", ".join(instance_segment_attrs)):
inst_dict = instance_infos[iseg.instance_info_id]
inst1 = inst_dict["inst1"]
begin_at = utils.str_to_datetime(iseg.begin_at)
end_at = utils.str_to_datetime(iseg.end_at)
inst_dict["begin_at"] = (min(inst_dict["begin_at"], begin_at)
if "begin_at" in inst_dict else begin_at)
try:
prev = inst_dict["end_at"]
except KeyError:
inst_dict["end_at"] = end_at
else:
inst_dict["end_at"] = (max(prev, end_at) if prev else None)
for rtype in instance_resources:
seg = Segment(resource_id=inst_dict[rtype + "_id"].id,
cost=getattr(inst1, rtype) * tariffs.get(rtype, 1),
begin_at=begin_at,
end_at=end_at)
db.session.add(seg)
for inst_dict in instance_infos.values():
seg = Segment(resource_id=inst_dict["inst2"].id,
cost=tariffs.get("nova/instance", 0),
begin_at=inst_dict.get("begin_at", None),
end_at=inst_dict.get("end_at", None))
db.session.add(seg)
db.session.commit()
def migrate_glance():
client = global_conf.clients.image
tariffs = db_api.tariff_map()
acc_man = AccountManager()
images = client.images.list()
for img1 in images:
if not img1.owner:
LOG.debug("image %s has no owner" %
img1.id)
continue
account_id = acc_man.get_or_create(img1.owner).id
img2 = db_api.resource_get_or_create(
account_id, None, None,
ResourceTypes.Image,
img1.id)
if check_skip_on_exists(img2):
continue
LOG.debug("adding info for image %s (name=%s)" % (img2.id, img2.name))
seg = Segment(
resource_id=img2.id,
cost=img1.size * tariffs.get(ResourceTypes.Image, 1) /
(1024.0 ** 3),
begin_at=utils.str_to_datetime(img1.created_at),
end_at=utils.str_to_datetime(img1.deleted_at))
db.session.add(seg)
db.session.commit()
def migrate_nova():
client = global_conf.clients.compute
tariffs = db_api.tariff_map()
acc_man = AccountManager()
flavors = {}
for flav in client.flavors.list():
flavors[str(flav.id)] = flav
deleted_flavors = set()
counter = 0
for deleted in 0, 1:
inst1_list = client.servers.list(
detailed=True,
search_opts={"deleted": deleted, "all_tenants": 1})
for inst1 in inst1_list:
acc_id = acc_man.get_or_create(inst1.tenant_id).id
inst2 = db_api.resource_get_or_create(
acc_id, None, None,
ResourceTypes.Instance,
inst1.id)
if check_skip_on_exists(inst2):
continue
try:
flav_id = str(inst1.flavor["id"])
flav = flavors[flav_id]
except KeyError:
LOG.error("cannot add info for instance %s (name=%s) "
"flavor %s is not found (perhaps it was deleted" %
(inst2.id, inst2.name, flav_id))
continue
LOG.debug("adding info for instance %s (name=%s)" % (inst2.id, inst2.name))
begin_at = utils.str_to_datetime(inst1.created)
end_at = utils.str_to_datetime(inst1.updated) if deleted else None
for nova, billing in flavor_map.iteritems():
child = db_api.resource_get_or_create(
acc_id, None, inst2.id,
billing,
None)
seg = Segment(
resource_id=child.id,
cost=getattr(flav, nova) * tariffs.get(billing, 1),
begin_at=begin_at,
end_at=end_at)
db.session.add(seg)
seg = Segment(
resource_id=inst2.id,
cost=tariffs.get(ResourceTypes.Instance, 0),
begin_at=begin_at,
end_at=end_at)
db.session.add(seg)
counter += 1
if counter % 32 == 0:
db.session.commit()
db.session.commit()
def get_period():
if not request.args.has_key("time_period"):
if "period_start" in request.args:
period_start = utils.str_to_datetime(request.args["period_start"])
try:
period_end = utils.str_to_datetime(request.args["period_end"])
except KeyError:
raise BadRequest(description="period_end is request.ired")
if not (period_start and period_end):
raise BadRequest(
description=
"date should be in ISO 8601 format of YYYY-MM-DDThh:mm:ssZ"
)
if period_start >= period_end:
raise BadRequest(
description="period_start must be less than period_end")
return period_start, period_end
else:
now = utils.now()
date_args = (now.year, now.month, 1)
date_incr = 1
else:
time_period_splitted = request.args["time_period"].split("-", 2)
date_args = [1, 1, 1]
for i in xrange(min(2, len(time_period_splitted))):
try:
date_args[i] = int(time_period_splitted[i])
except ValueError:
raise BadRequest(description="invalid time_period `%s'" %
request.args["time_period"])
date_incr = len(time_period_splitted) - 1
period_start = datetime.datetime(*date_args)
if date_incr == 2:
period_end = period_start + datetime.timedelta(days=1)
else:
year, month, day = date_args
if date_incr == 1:
month += 1
if month > 12:
month = 1
year += 1
else:
year += 1
period_end = datetime.datetime(year=year, month=month, day=day)
return period_start, period_end
def get_period():
if not request.args.has_key("time_period"):
if "period_start" in request.args:
period_start = utils.str_to_datetime(request.args["period_start"])
try:
period_end = utils.str_to_datetime(request.args["period_end"])
except KeyError:
raise BadRequest(description="period_end is request.ired")
if not (period_start and period_end):
raise BadRequest(
description="date should be in ISO 8601 format of YYYY-MM-DDThh:mm:ssZ")
if period_start >= period_end:
raise BadRequest(
description="period_start must be less than period_end")
return period_start, period_end
else:
now = utils.now()
date_args = (now.year, now.month, 1)
date_incr = 1
else:
time_period_splitted = request.args["time_period"].split("-", 2)
date_args = [1, 1, 1]
for i in xrange(min(2, len(time_period_splitted))):
try:
date_args[i] = int(time_period_splitted[i])
except ValueError:
raise BadRequest(
description="invalid time_period `%s'" % request.args["time_period"])
date_incr = len(time_period_splitted) - 1
period_start = datetime.datetime(*date_args)
if date_incr == 2:
period_end = period_start + datetime.timedelta(days=1)
else:
year, month, day = date_args
if date_incr == 1:
month += 1
if month > 12:
month = 1
year += 1
else:
year += 1
period_end = datetime.datetime(year=year, month=month, day=day)
return period_start, period_end
def migrate_instances(old_db_url):
engine1 = create_engine(old_db_url)
tariffs = db_api.tariff_map()
instance_resources = ("local_gb", "memory_mb", "vcpus")
instance_info_attrs = (
"id", "instance_id", "project_id",
"local_gb", "memory_mb", "vcpus")
instance_segment_attrs = (
"id", "instance_info_id",
"segment_type", "begin_at",
"end_at")
instance_infos = {}
accounts = {}
for inst1 in engine1.execute(
"select distinct project_id from billing_instance_info"):
accounts[inst1.project_id] = \
db_api.account_get_or_create(inst1.project_id).id
for inst1 in engine1.execute(
"select %s from billing_instance_info" %
", ".join(instance_info_attrs)):
account_id = accounts[inst1.project_id]
inst2 = db_api.resource_get_or_create(
account_id, None,
ResourceTypes.Instance,
inst1.instance_id
)
inst_dict = {
"inst1": inst1,
"inst2": inst2,
}
for rtype in instance_resources:
inst_dict[rtype + "_id"] = db_api.resource_get_or_create(
account_id, inst2.id,
rtype,
None
)
instance_infos[inst1.id] = inst_dict
for iseg in engine1.execute(
"select %s from billing_instance_segment" %
", ".join(instance_segment_attrs)):
inst_dict = instance_infos[iseg.instance_info_id]
inst1 = inst_dict["inst1"]
begin_at = utils.str_to_datetime(iseg.begin_at)
end_at = utils.str_to_datetime(iseg.end_at)
inst_dict["begin_at"] = (min(inst_dict["begin_at"], begin_at)
if "begin_at" in inst_dict else begin_at)
try:
prev = inst_dict["end_at"]
except KeyError:
inst_dict["end_at"] = end_at
else:
inst_dict["end_at"] = (
max(prev, end_at) if prev
else None)
for rtype in instance_resources:
seg = Segment(
resource_id=inst_dict[rtype + "_id"].id,
cost=getattr(inst1, rtype) * tariffs.get(rtype, 1),
begin_at=begin_at,
end_at=end_at)
db.session.add(seg)
for inst_dict in instance_infos.values():
seg = Segment(
resource_id=inst_dict["inst2"].id,
cost=tariffs.get("nova/instance", 0),
begin_at=inst_dict.get("begin_at", None),
end_at=inst_dict.get("end_at", None))
db.session.add(seg)
db.session.commit()
def check_and_get_datatime(rj):
ret = utils.str_to_datetime(rj.get("datetime", None))
if not ret:
raise BadRequest(
description="valid datetime must be specified")
return ret
def check_and_get_datatime(rj):
ret = utils.str_to_datetime(rj.get("datetime", None))
if not ret:
raise BadRequest(description="valid datetime must be specified")
return ret
def images_on_interval(period_start, period_stop, auth_tok, tenant_id=None):
"""
Retrieve images statistics for the given interval [``period_start``, ``period_stop``].
``tenant_id=None`` means all projects.
Example of the returned value:
.. code-block:: python
{
"1": {
12: {
"created_at": datetime.datetime(2011, 1, 1),
"destroyed_at": datetime.datetime(2011, 1, 2),
"name": "Gentoo initrd",
"usage": {"local_gb": 0.1},
},
14: {
"created_at": datetime.datetime(2011, 1, 4),
"destroyed_at": datetime.datetime(2011, 2, 1),
"name": "Ubuntu vmlinuz",
"usage": {"local_gb": 2.5},
},
},
"2": {
24: {
"created_at": datetime.datetime(2011, 1, 1),
"destroyed_at": datetime.datetime(2011, 1, 2),
"name": "RHEL vmlinuz",
"usage": {"local_gb": 6.1},
},
}
}
:returns: a dictionary where keys are project ids and values are project statistics.
"""
glance_host, glance_port = pick_glance_api_server()
client = glance_client.Client(glance_host, glance_port, auth_tok=auth_tok)
images = client.get_images_detailed(filters={"is_public": "none"})
if tenant_id:
images = [image for image in images if image["owner"] == tenant_id]
else:
images = [image for image in images if image["owner"] is not None]
report_by_id = {}
now = utils.now()
for image in images:
created_at = utils.str_to_datetime(image["created_at"])
if created_at >= period_stop:
continue
deleted_at = utils.str_to_datetime(image["deleted_at"])
if deleted_at and deleted_at <= period_start:
continue
lifetime = utils.total_seconds(
min(deleted_at or now, period_stop) -
max(created_at, period_start))
if lifetime < 0:
lifetime = 0
try:
tenant_statistics = report_by_id[image["owner"]]
except KeyError:
tenant_statistics = {}
report_by_id[image["owner"]] = tenant_statistics
tenant_statistics[image["id"]] = {
"name": image["name"],
"created_at": created_at,
"destroyed_at": deleted_at,
"usage": {"local_gb": image["size"] * lifetime / 2 ** 30}
}
return report_by_id