Пример #1
0
def migrate_images(glance_url):
    glance_client = client.RestClient()
    glance_client.auth_headers = {"x-auth-token": global_conf.admin_token}
    glance_client.management_url = glance_url
    
    tariffs = db_api.tariff_map()
    accounts = {}
    images = json.loads(glance_client.get("/images/detail"))["images"]
    for project_id in (img1["owner"] for img1 in images if img1["owner"]):
        accounts[project_id] = \
            db_api.account_get_or_create(project_id).id
    
    for img1 in images:
        if not img1["owner"]:
            continue
        account_id = accounts[img1["owner"]]
        img2 = db_api.resource_get_or_create(
            account_id, None,
            ResourceTypes.Image,
            img1["id"]
        )
        seg = Segment(
            resource_id=img2.id,
            cost=img1["size"] * tariffs.get(ResourceTypes.Image, 1) / (1024.0 ** 3),
            begin_at=utils.str_to_datetime(img1["created_at"]),
            end_at=utils.str_to_datetime(img1["deleted_at"]))
        db.session.add(seg)

    db.session.commit()
Пример #2
0
def migrate_images(glance_url):
    glance_client = client.RestClient()
    glance_client.auth_headers = {"x-auth-token": global_conf.admin_token}
    glance_client.management_url = glance_url

    tariffs = db_api.tariff_map()
    accounts = {}
    images = json.loads(glance_client.get("/images/detail"))["images"]
    for project_id in (img1["owner"] for img1 in images if img1["owner"]):
        accounts[project_id] = \
            db_api.account_get_or_create(project_id).id

    for img1 in images:
        if not img1["owner"]:
            continue
        account_id = accounts[img1["owner"]]
        img2 = db_api.resource_get_or_create(account_id, None,
                                             ResourceTypes.Image, img1["id"])
        seg = Segment(resource_id=img2.id,
                      cost=img1["size"] * tariffs.get(ResourceTypes.Image, 1) /
                      (1024.0**3),
                      begin_at=utils.str_to_datetime(img1["created_at"]),
                      end_at=utils.str_to_datetime(img1["deleted_at"]))
        db.session.add(seg)

    db.session.commit()
Пример #3
0
def migrate_instances(old_db_url):
    engine1 = create_engine(old_db_url)

    tariffs = db_api.tariff_map()
    instance_resources = ("local_gb", "memory_mb", "vcpus")
    instance_info_attrs = ("id", "instance_id", "project_id", "local_gb",
                           "memory_mb", "vcpus")
    instance_segment_attrs = ("id", "instance_info_id", "segment_type",
                              "begin_at", "end_at")
    instance_infos = {}
    accounts = {}
    for inst1 in engine1.execute(
            "select distinct project_id from billing_instance_info"):
        accounts[inst1.project_id] = \
            db_api.account_get_or_create(inst1.project_id).id

    for inst1 in engine1.execute("select %s from billing_instance_info" %
                                 ", ".join(instance_info_attrs)):
        account_id = accounts[inst1.project_id]
        inst2 = db_api.resource_get_or_create(account_id, None,
                                              ResourceTypes.Instance,
                                              inst1.instance_id)
        inst_dict = {
            "inst1": inst1,
            "inst2": inst2,
        }
        for rtype in instance_resources:
            inst_dict[rtype + "_id"] = db_api.resource_get_or_create(
                account_id, inst2.id, rtype, None)
        instance_infos[inst1.id] = inst_dict

    for iseg in engine1.execute("select %s from billing_instance_segment" %
                                ", ".join(instance_segment_attrs)):
        inst_dict = instance_infos[iseg.instance_info_id]
        inst1 = inst_dict["inst1"]
        begin_at = utils.str_to_datetime(iseg.begin_at)
        end_at = utils.str_to_datetime(iseg.end_at)
        inst_dict["begin_at"] = (min(inst_dict["begin_at"], begin_at)
                                 if "begin_at" in inst_dict else begin_at)
        try:
            prev = inst_dict["end_at"]
        except KeyError:
            inst_dict["end_at"] = end_at
        else:
            inst_dict["end_at"] = (max(prev, end_at) if prev else None)
        for rtype in instance_resources:
            seg = Segment(resource_id=inst_dict[rtype + "_id"].id,
                          cost=getattr(inst1, rtype) * tariffs.get(rtype, 1),
                          begin_at=begin_at,
                          end_at=end_at)
            db.session.add(seg)

    for inst_dict in instance_infos.values():
        seg = Segment(resource_id=inst_dict["inst2"].id,
                      cost=tariffs.get("nova/instance", 0),
                      begin_at=inst_dict.get("begin_at", None),
                      end_at=inst_dict.get("end_at", None))
        db.session.add(seg)

    db.session.commit()
Пример #4
0
def migrate_glance():
    client = global_conf.clients.image
    tariffs = db_api.tariff_map()
    acc_man = AccountManager()

    images = client.images.list()
    for img1 in images:
        if not img1.owner:
            LOG.debug("image %s has no owner" %
                      img1.id)
            continue
        account_id = acc_man.get_or_create(img1.owner).id
        img2 = db_api.resource_get_or_create(
            account_id, None, None,
            ResourceTypes.Image,
            img1.id)
        if check_skip_on_exists(img2):
            continue
        LOG.debug("adding info for image %s (name=%s)" % (img2.id, img2.name))
        seg = Segment(
            resource_id=img2.id,
            cost=img1.size * tariffs.get(ResourceTypes.Image, 1) /
            (1024.0 ** 3),
            begin_at=utils.str_to_datetime(img1.created_at),
            end_at=utils.str_to_datetime(img1.deleted_at))
        db.session.add(seg)

    db.session.commit()
Пример #5
0
def migrate_nova():
    client = global_conf.clients.compute
    tariffs = db_api.tariff_map()
    acc_man = AccountManager()
    flavors = {}
    for flav in client.flavors.list():
        flavors[str(flav.id)] = flav
    deleted_flavors = set()
    counter = 0
    for deleted in 0, 1:
        inst1_list = client.servers.list(
            detailed=True,
            search_opts={"deleted": deleted, "all_tenants": 1})
        for inst1 in inst1_list:
            acc_id = acc_man.get_or_create(inst1.tenant_id).id
            inst2 = db_api.resource_get_or_create(
                acc_id, None, None,
                ResourceTypes.Instance,
                inst1.id)
            if check_skip_on_exists(inst2):
                continue

            try:
                flav_id = str(inst1.flavor["id"])
                flav = flavors[flav_id]
            except KeyError:
                LOG.error("cannot add info for instance %s (name=%s) "
                          "flavor %s is not found (perhaps it was deleted" %
                          (inst2.id, inst2.name, flav_id))
                continue
            LOG.debug("adding info for instance %s (name=%s)" % (inst2.id, inst2.name))
            begin_at = utils.str_to_datetime(inst1.created)
            end_at = utils.str_to_datetime(inst1.updated) if deleted else None
            for nova, billing in flavor_map.iteritems():
                child = db_api.resource_get_or_create(
                    acc_id, None, inst2.id,
                    billing,
                    None)
                seg = Segment(
                    resource_id=child.id,
                    cost=getattr(flav, nova) * tariffs.get(billing, 1),
                    begin_at=begin_at,
                    end_at=end_at)
                db.session.add(seg)
            seg = Segment(
                resource_id=inst2.id,
                cost=tariffs.get(ResourceTypes.Instance, 0),
                begin_at=begin_at,
                end_at=end_at)
            db.session.add(seg)
            counter += 1
            if counter % 32 == 0:
                db.session.commit()
    db.session.commit()
Пример #6
0
def get_period():
    if not request.args.has_key("time_period"):
        if "period_start" in request.args:
            period_start = utils.str_to_datetime(request.args["period_start"])
            try:
                period_end = utils.str_to_datetime(request.args["period_end"])
            except KeyError:
                raise BadRequest(description="period_end is request.ired")
            if not (period_start and period_end):
                raise BadRequest(
                    description=
                    "date should be in ISO 8601 format of YYYY-MM-DDThh:mm:ssZ"
                )
            if period_start >= period_end:
                raise BadRequest(
                    description="period_start must be less than period_end")
            return period_start, period_end
        else:
            now = utils.now()
            date_args = (now.year, now.month, 1)
            date_incr = 1
    else:
        time_period_splitted = request.args["time_period"].split("-", 2)
        date_args = [1, 1, 1]
        for i in xrange(min(2, len(time_period_splitted))):
            try:
                date_args[i] = int(time_period_splitted[i])
            except ValueError:
                raise BadRequest(description="invalid time_period `%s'" %
                                 request.args["time_period"])
        date_incr = len(time_period_splitted) - 1

    period_start = datetime.datetime(*date_args)
    if date_incr == 2:
        period_end = period_start + datetime.timedelta(days=1)
    else:
        year, month, day = date_args
        if date_incr == 1:
            month += 1
            if month > 12:
                month = 1
                year += 1
        else:
            year += 1
        period_end = datetime.datetime(year=year, month=month, day=day)
    return period_start, period_end
Пример #7
0
def get_period():
    if not request.args.has_key("time_period"):
        if "period_start" in request.args:
            period_start = utils.str_to_datetime(request.args["period_start"])
            try:
                period_end = utils.str_to_datetime(request.args["period_end"])
            except KeyError:
                raise BadRequest(description="period_end is request.ired")
            if not (period_start and period_end):
                raise BadRequest(
                    description="date should be in ISO 8601 format of YYYY-MM-DDThh:mm:ssZ")
            if period_start >= period_end:
                raise BadRequest(
                    description="period_start must be less than period_end")
            return period_start, period_end
        else:
            now = utils.now()
            date_args = (now.year, now.month, 1)
            date_incr = 1
    else:
        time_period_splitted = request.args["time_period"].split("-", 2)
        date_args = [1, 1, 1]
        for i in xrange(min(2, len(time_period_splitted))):
            try:
                date_args[i] = int(time_period_splitted[i])
            except ValueError:
                raise BadRequest(
                    description="invalid time_period `%s'" % request.args["time_period"])
        date_incr = len(time_period_splitted) - 1

    period_start = datetime.datetime(*date_args)
    if date_incr == 2:
        period_end = period_start + datetime.timedelta(days=1)
    else:
        year, month, day = date_args
        if date_incr == 1:
            month += 1
            if month > 12:
                month = 1
                year += 1
        else:
            year += 1
        period_end = datetime.datetime(year=year, month=month, day=day)
    return period_start, period_end
Пример #8
0
def migrate_instances(old_db_url):
    engine1 = create_engine(old_db_url)

    tariffs = db_api.tariff_map()
    instance_resources = ("local_gb", "memory_mb", "vcpus")
    instance_info_attrs = (
        "id", "instance_id", "project_id",
        "local_gb", "memory_mb", "vcpus")
    instance_segment_attrs = (
        "id", "instance_info_id",
        "segment_type", "begin_at",
        "end_at")
    instance_infos = {}
    accounts = {}
    for inst1 in engine1.execute(
        "select distinct project_id from billing_instance_info"):
        accounts[inst1.project_id] = \
            db_api.account_get_or_create(inst1.project_id).id
    
    for inst1 in engine1.execute(
        "select %s from billing_instance_info" %
        ", ".join(instance_info_attrs)):
        account_id = accounts[inst1.project_id]
        inst2 = db_api.resource_get_or_create(
            account_id, None,
            ResourceTypes.Instance,
            inst1.instance_id
        )
        inst_dict = {
            "inst1": inst1,
            "inst2": inst2,
        }
        for rtype in instance_resources:
            inst_dict[rtype + "_id"] = db_api.resource_get_or_create(
                account_id, inst2.id, 
                rtype,
                None
            )
        instance_infos[inst1.id] = inst_dict
    
    for iseg in engine1.execute(
        "select %s from billing_instance_segment" %
        ", ".join(instance_segment_attrs)):
        inst_dict = instance_infos[iseg.instance_info_id]
        inst1 = inst_dict["inst1"]
        begin_at = utils.str_to_datetime(iseg.begin_at)
        end_at = utils.str_to_datetime(iseg.end_at)
        inst_dict["begin_at"] = (min(inst_dict["begin_at"], begin_at)
                                 if "begin_at" in inst_dict else begin_at)
        try:
            prev = inst_dict["end_at"]
        except KeyError:
            inst_dict["end_at"] = end_at
        else:
            inst_dict["end_at"] = (
                max(prev, end_at) if prev
                else None)
        for rtype in instance_resources:
            seg = Segment(
                resource_id=inst_dict[rtype + "_id"].id,
                cost=getattr(inst1, rtype) * tariffs.get(rtype, 1),
                begin_at=begin_at,
                end_at=end_at)
            db.session.add(seg)

    for inst_dict in instance_infos.values():
        seg = Segment(
            resource_id=inst_dict["inst2"].id,
            cost=tariffs.get("nova/instance", 0),
            begin_at=inst_dict.get("begin_at", None),
            end_at=inst_dict.get("end_at", None))
        db.session.add(seg)

    db.session.commit()
Пример #9
0
def check_and_get_datatime(rj):
    ret = utils.str_to_datetime(rj.get("datetime", None))
    if not ret:
        raise BadRequest(
            description="valid datetime must be specified")
    return ret
Пример #10
0
def check_and_get_datatime(rj):
    ret = utils.str_to_datetime(rj.get("datetime", None))
    if not ret:
        raise BadRequest(description="valid datetime must be specified")
    return ret
Пример #11
0
def images_on_interval(period_start, period_stop, auth_tok, tenant_id=None):
    """
    Retrieve images statistics for the given interval [``period_start``, ``period_stop``]. 
    ``tenant_id=None`` means all projects.

    Example of the returned value:

    .. code-block:: python

        {
            "1": {
                12: {
                    "created_at": datetime.datetime(2011, 1, 1),
                    "destroyed_at": datetime.datetime(2011, 1, 2),
                    "name": "Gentoo initrd",
                    "usage": {"local_gb": 0.1},
                },
                14: {
                    "created_at": datetime.datetime(2011, 1, 4),
                    "destroyed_at": datetime.datetime(2011, 2, 1),
                    "name": "Ubuntu vmlinuz",
                    "usage": {"local_gb": 2.5},
                },
            },
            "2": {
                24: {
                    "created_at": datetime.datetime(2011, 1, 1),
                    "destroyed_at": datetime.datetime(2011, 1, 2),
                    "name": "RHEL vmlinuz",
                    "usage": {"local_gb": 6.1},
                },
            }
        }

    :returns: a dictionary where keys are project ids and values are project statistics.
    """

    glance_host, glance_port = pick_glance_api_server()
    client = glance_client.Client(glance_host, glance_port, auth_tok=auth_tok)
    images = client.get_images_detailed(filters={"is_public": "none"})
    if tenant_id:
        images = [image for image in images if image["owner"] == tenant_id]
    else:
        images = [image for image in images if image["owner"] is not None]
    report_by_id = {}
    now = utils.now()
    for image in images:
        created_at = utils.str_to_datetime(image["created_at"])
        if created_at >= period_stop:
            continue
        deleted_at = utils.str_to_datetime(image["deleted_at"])
        if deleted_at and deleted_at <= period_start:
            continue
        lifetime = utils.total_seconds(
            min(deleted_at or now, period_stop) -
            max(created_at, period_start))
        if lifetime < 0:
            lifetime = 0
        try:
            tenant_statistics = report_by_id[image["owner"]]
        except KeyError:
            tenant_statistics = {}
            report_by_id[image["owner"]] = tenant_statistics
        tenant_statistics[image["id"]] = {
            "name": image["name"],
            "created_at": created_at,
            "destroyed_at": deleted_at,
            "usage": {"local_gb": image["size"] * lifetime / 2 ** 30}
        }

    return report_by_id