def migrate_instances(old_db_url): engine1 = create_engine(old_db_url) tariffs = db_api.tariff_map() instance_resources = ("local_gb", "memory_mb", "vcpus") instance_info_attrs = ("id", "instance_id", "project_id", "local_gb", "memory_mb", "vcpus") instance_segment_attrs = ("id", "instance_info_id", "segment_type", "begin_at", "end_at") instance_infos = {} accounts = {} for inst1 in engine1.execute( "select distinct project_id from billing_instance_info"): accounts[inst1.project_id] = \ db_api.account_get_or_create(inst1.project_id).id for inst1 in engine1.execute("select %s from billing_instance_info" % ", ".join(instance_info_attrs)): account_id = accounts[inst1.project_id] inst2 = db_api.resource_get_or_create(account_id, None, ResourceTypes.Instance, inst1.instance_id) inst_dict = { "inst1": inst1, "inst2": inst2, } for rtype in instance_resources: inst_dict[rtype + "_id"] = db_api.resource_get_or_create( account_id, inst2.id, rtype, None) instance_infos[inst1.id] = inst_dict for iseg in engine1.execute("select %s from billing_instance_segment" % ", ".join(instance_segment_attrs)): inst_dict = instance_infos[iseg.instance_info_id] inst1 = inst_dict["inst1"] begin_at = utils.str_to_datetime(iseg.begin_at) end_at = utils.str_to_datetime(iseg.end_at) inst_dict["begin_at"] = (min(inst_dict["begin_at"], begin_at) if "begin_at" in inst_dict else begin_at) try: prev = inst_dict["end_at"] except KeyError: inst_dict["end_at"] = end_at else: inst_dict["end_at"] = (max(prev, end_at) if prev else None) for rtype in instance_resources: seg = Segment(resource_id=inst_dict[rtype + "_id"].id, cost=getattr(inst1, rtype) * tariffs.get(rtype, 1), begin_at=begin_at, end_at=end_at) db.session.add(seg) for inst_dict in instance_infos.values(): seg = Segment(resource_id=inst_dict["inst2"].id, cost=tariffs.get("nova/instance", 0), begin_at=inst_dict.get("begin_at", None), end_at=inst_dict.get("end_at", None)) db.session.add(seg) db.session.commit()
def migrate_nova(): client = global_conf.clients.compute tariffs = db_api.tariff_map() acc_man = AccountManager() flavors = {} for flav in client.flavors.list(): flavors[str(flav.id)] = flav deleted_flavors = set() counter = 0 for deleted in 0, 1: inst1_list = client.servers.list( detailed=True, search_opts={"deleted": deleted, "all_tenants": 1}) for inst1 in inst1_list: acc_id = acc_man.get_or_create(inst1.tenant_id).id inst2 = db_api.resource_get_or_create( acc_id, None, None, ResourceTypes.Instance, inst1.id) if check_skip_on_exists(inst2): continue try: flav_id = str(inst1.flavor["id"]) flav = flavors[flav_id] except KeyError: LOG.error("cannot add info for instance %s (name=%s) " "flavor %s is not found (perhaps it was deleted" % (inst2.id, inst2.name, flav_id)) continue LOG.debug("adding info for instance %s (name=%s)" % (inst2.id, inst2.name)) begin_at = utils.str_to_datetime(inst1.created) end_at = utils.str_to_datetime(inst1.updated) if deleted else None for nova, billing in flavor_map.iteritems(): child = db_api.resource_get_or_create( acc_id, None, inst2.id, billing, None) seg = Segment( resource_id=child.id, cost=getattr(flav, nova) * tariffs.get(billing, 1), begin_at=begin_at, end_at=end_at) db.session.add(seg) seg = Segment( resource_id=inst2.id, cost=tariffs.get(ResourceTypes.Instance, 0), begin_at=begin_at, end_at=end_at) db.session.add(seg) counter += 1 if counter % 32 == 0: db.session.commit() db.session.commit()
def migrate_images(glance_url): glance_client = client.RestClient() glance_client.auth_headers = {"x-auth-token": global_conf.admin_token} glance_client.management_url = glance_url tariffs = db_api.tariff_map() accounts = {} images = json.loads(glance_client.get("/images/detail"))["images"] for project_id in (img1["owner"] for img1 in images if img1["owner"]): accounts[project_id] = \ db_api.account_get_or_create(project_id).id for img1 in images: if not img1["owner"]: continue account_id = accounts[img1["owner"]] img2 = db_api.resource_get_or_create( account_id, None, ResourceTypes.Image, img1["id"] ) seg = Segment( resource_id=img2.id, cost=img1["size"] * tariffs.get(ResourceTypes.Image, 1) / (1024.0 ** 3), begin_at=utils.str_to_datetime(img1["created_at"]), end_at=utils.str_to_datetime(img1["deleted_at"])) db.session.add(seg) db.session.commit()
def migrate_images(glance_url): glance_client = client.RestClient() glance_client.auth_headers = {"x-auth-token": global_conf.admin_token} glance_client.management_url = glance_url tariffs = db_api.tariff_map() accounts = {} images = json.loads(glance_client.get("/images/detail"))["images"] for project_id in (img1["owner"] for img1 in images if img1["owner"]): accounts[project_id] = \ db_api.account_get_or_create(project_id).id for img1 in images: if not img1["owner"]: continue account_id = accounts[img1["owner"]] img2 = db_api.resource_get_or_create(account_id, None, ResourceTypes.Image, img1["id"]) seg = Segment(resource_id=img2.id, cost=img1["size"] * tariffs.get(ResourceTypes.Image, 1) / (1024.0**3), begin_at=utils.str_to_datetime(img1["created_at"]), end_at=utils.str_to_datetime(img1["deleted_at"])) db.session.add(seg) db.session.commit()
def migrate_glance(): client = global_conf.clients.image tariffs = db_api.tariff_map() acc_man = AccountManager() images = client.images.list() for img1 in images: if not img1.owner: LOG.debug("image %s has no owner" % img1.id) continue account_id = acc_man.get_or_create(img1.owner).id img2 = db_api.resource_get_or_create( account_id, None, None, ResourceTypes.Image, img1.id) if check_skip_on_exists(img2): continue LOG.debug("adding info for image %s (name=%s)" % (img2.id, img2.name)) seg = Segment( resource_id=img2.id, cost=img1.size * tariffs.get(ResourceTypes.Image, 1) / (1024.0 ** 3), begin_at=utils.str_to_datetime(img1.created_at), end_at=utils.str_to_datetime(img1.deleted_at)) db.session.add(seg) db.session.commit()
def migrate_instances(old_db_url): engine1 = create_engine(old_db_url) tariffs = db_api.tariff_map() instance_resources = ("local_gb", "memory_mb", "vcpus") instance_info_attrs = ( "id", "instance_id", "project_id", "local_gb", "memory_mb", "vcpus") instance_segment_attrs = ( "id", "instance_info_id", "segment_type", "begin_at", "end_at") instance_infos = {} accounts = {} for inst1 in engine1.execute( "select distinct project_id from billing_instance_info"): accounts[inst1.project_id] = \ db_api.account_get_or_create(inst1.project_id).id for inst1 in engine1.execute( "select %s from billing_instance_info" % ", ".join(instance_info_attrs)): account_id = accounts[inst1.project_id] inst2 = db_api.resource_get_or_create( account_id, None, ResourceTypes.Instance, inst1.instance_id ) inst_dict = { "inst1": inst1, "inst2": inst2, } for rtype in instance_resources: inst_dict[rtype + "_id"] = db_api.resource_get_or_create( account_id, inst2.id, rtype, None ) instance_infos[inst1.id] = inst_dict for iseg in engine1.execute( "select %s from billing_instance_segment" % ", ".join(instance_segment_attrs)): inst_dict = instance_infos[iseg.instance_info_id] inst1 = inst_dict["inst1"] begin_at = utils.str_to_datetime(iseg.begin_at) end_at = utils.str_to_datetime(iseg.end_at) inst_dict["begin_at"] = (min(inst_dict["begin_at"], begin_at) if "begin_at" in inst_dict else begin_at) try: prev = inst_dict["end_at"] except KeyError: inst_dict["end_at"] = end_at else: inst_dict["end_at"] = ( max(prev, end_at) if prev else None) for rtype in instance_resources: seg = Segment( resource_id=inst_dict[rtype + "_id"].id, cost=getattr(inst1, rtype) * tariffs.get(rtype, 1), begin_at=begin_at, end_at=end_at) db.session.add(seg) for inst_dict in instance_infos.values(): seg = Segment( resource_id=inst_dict["inst2"].id, cost=tariffs.get("nova/instance", 0), begin_at=inst_dict.get("begin_at", None), end_at=inst_dict.get("end_at", None)) db.session.add(seg) db.session.commit()