def compose__import_rpms(request, release_id, composeinfo, rpm_manifest):
    release_obj = release_models.Release.objects.get(release_id=release_id)

    ci = common_hacks.composeinfo_from_str(composeinfo)
    rm = common_hacks.rpms_from_str(rpm_manifest)

    compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6], ci.compose.date[6:])
    compose_type = models.ComposeType.objects.get(name=ci.compose.type)
    acceptance_status = models.ComposeAcceptanceTestingState.objects.get(name='untested')
    compose_obj, _ = models.Compose.objects.get_or_create(
        release=release_obj,
        compose_id=ci.compose.id,
        compose_date=compose_date,
        compose_type=compose_type,
        compose_respin=ci.compose.respin,
        compose_label=ci.compose.label or None,
        acceptance_testing=acceptance_status,
    )

    rpms_in_db = {}
    qs = package_models.RPM.objects.all()
    for rpm in qs.iterator():
        key = "%s-%s:%s-%s.%s" % (rpm.name, rpm.epoch, rpm.version, rpm.release, rpm.arch)
        rpms_in_db[key] = rpm.id

    cursor = connection.cursor()
    for variant in ci.get_variants(recursive=True):
        _link_compose_to_integrated_product(compose_obj, variant)
        variant_type = release_models.VariantType.objects.get(name=variant.type)
        variant_obj, _ = models.Variant.objects.get_or_create(compose=compose_obj,
                                                              variant_id=variant.id,
                                                              variant_uid=variant.uid,
                                                              variant_name=variant.name,
                                                              variant_type=variant_type)
        for arch in variant.arches:
            arch_obj = common_models.Arch.objects.get(name=arch)
            var_arch_obj, _ = models.VariantArch.objects.get_or_create(arch=arch_obj,
                                                                       variant=variant_obj)

            compose_rpms_in_db = set()
            qs = models.ComposeRPM.objects.filter(variant_arch=var_arch_obj).values_list('variant_arch_id',
                                                                                         'rpm_id')
            for (variant_arch_id, rpm_id) in qs.iterator():
                key = "%s/%s" % (variant_arch_id, rpm_id)
                compose_rpms_in_db.add(key)

            sources = set()
            for srpm_nevra, rpms in rm.rpms.get(variant.uid, {}).get(arch, {}).iteritems():
                sources.add(srpm_nevra)
                for rpm_nevra, rpm_data in rpms.iteritems():
                    path, filename = os.path.split(rpm_data['path'])
                    rpm_id = get_or_insert_rpm(rpms_in_db, cursor, rpm_nevra, srpm_nevra, filename)
                    sigkey_id = common_models.SigKey.get_cached_id(rpm_data["sigkey"], create=True)
                    path_id = models.Path.get_cached_id(path, create=True)
                    content_category = rpm_data["category"]
                    content_category_id = repository_models.ContentCategory.get_cached_id(content_category)
                    insert_compose_rpms_if_nonexist(compose_rpms_in_db, cursor,
                                                    var_arch_obj.id, rpm_id,
                                                    content_category_id, sigkey_id, path_id)
def compose__import_images(request, release_id, composeinfo, image_manifest):
    release_obj = release_models.Release.objects.get(release_id=release_id)

    ci = common_hacks.composeinfo_from_str(composeinfo)
    im = common_hacks.images_from_str(image_manifest)

    compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6], ci.compose.date[6:])
    compose_type = models.ComposeType.objects.get(name=ci.compose.type)
    compose_obj, _ = models.Compose.objects.get_or_create(
        release=release_obj,
        compose_id=ci.compose.id,
        compose_date=compose_date,
        compose_type=compose_type,
        compose_respin=ci.compose.respin,
        compose_label=ci.compose.label or None,
    )

    for variant in ci.get_variants(recursive=True):
        _link_compose_to_integrated_product(compose_obj, variant)
        variant_type = release_models.VariantType.objects.get(name=variant.type)
        variant_obj, _ = models.Variant.objects.get_or_create(
            compose=compose_obj,
            variant_id=variant.id,
            variant_uid=variant.uid,
            variant_name=variant.name,
            variant_type=variant_type
        )
        for arch in variant.arches:
            arch_obj = common_models.Arch.objects.get(name=arch)
            var_arch_obj, created = models.VariantArch.objects.get_or_create(arch=arch_obj, variant=variant_obj)

            for i in im.images.get(variant.uid, {}).get(arch, []):
                # TODO: handle properly
                try:
                    image = package_models.Image.objects.get(file_name=os.path.basename(i.path), sha256=i.checksums["sha256"])
                except package_models.Image.DoesNotExist:
                    image = package_models.Image()
                image.file_name = os.path.basename(i.path)
                image.image_format_id = package_models.ImageFormat.get_cached_id(i.format)
                image.image_type_id = package_models.ImageType.get_cached_id(i.type)
                image.disc_number = i.disc_number
                image.disc_count = i.disc_count
                image.arch = i.arch
                image.mtime = i.mtime
                image.size = i.size
                image.bootable = i.bootable
                image.implant_md5 = i.implant_md5
                image.volume_id = i.volume_id
                image.md5 = i.checksums.get("md5", None)
                image.sha1 = i.checksums.get("sha1", None)
                image.sha256 = i.checksums.get("sha256", None)
                image.save()

                # TODO: path to ComposeImage
                mi, created = models.ComposeImage.objects.get_or_create(
                    variant_arch=var_arch_obj,
                    image=image)
def compose__import_images(request, release_id, composeinfo, image_manifest):
    release_obj = release_models.Release.objects.get(release_id=release_id)

    ci = common_hacks.composeinfo_from_str(composeinfo)
    im = common_hacks.images_from_str(image_manifest)

    compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6], ci.compose.date[6:])
    compose_type = models.ComposeType.objects.get(name=ci.compose.type)
    compose_obj, created = lib._logged_get_or_create(
        request, models.Compose,
        release=release_obj,
        compose_id=ci.compose.id,
        compose_date=compose_date,
        compose_type=compose_type,
        compose_respin=ci.compose.respin,
        compose_label=ci.compose.label or None,
    )
    if created and hasattr(request._request, '_messagings'):
        # add message
        _add_compose_create_msg(request, compose_obj)

    add_to_changelog = []
    imported_images = 0

    for variant in ci.get_variants(recursive=True):
        _link_compose_to_integrated_product(request, compose_obj, variant)
        variant_type = release_models.VariantType.objects.get(name=variant.type)
        variant_obj, created = models.Variant.objects.get_or_create(
            compose=compose_obj,
            variant_id=variant.id,
            variant_uid=variant.uid,
            variant_name=variant.name,
            variant_type=variant_type
        )
        if created:
            add_to_changelog.append(variant_obj)
        for arch in variant.arches:
            arch_obj = common_models.Arch.objects.get(name=arch)
            var_arch_obj, created = models.VariantArch.objects.get_or_create(arch=arch_obj, variant=variant_obj)

            for i in im.images.get(variant.uid, {}).get(arch, []):
                path, file_name = os.path.split(i.path)
                path_id = models.Path.get_cached_id(path, create=True)

                image, _ = package_models.Image.objects.get_or_create(
                    file_name=file_name, sha256=i.checksums["sha256"],
                    defaults={
                        'image_format_id': package_models.ImageFormat.get_cached_id(i.format),
                        'image_type_id': package_models.ImageType.get_cached_id(i.type),
                        'disc_number': i.disc_number,
                        'disc_count': i.disc_count,
                        'arch': i.arch,
                        'mtime': i.mtime,
                        'size': i.size,
                        'bootable': i.bootable,
                        'implant_md5': i.implant_md5,
                        'volume_id': i.volume_id,
                        'md5': i.checksums.get("md5", None),
                        'sha1': i.checksums.get("sha1", None),
                    }
                )

                mi, created = models.ComposeImage.objects.get_or_create(
                    variant_arch=var_arch_obj,
                    image=image,
                    path_id=path_id)
                imported_images += 1

    for obj in add_to_changelog:
        lib._maybe_log(request, True, obj)

    request.changeset.add('notice', 0, 'null',
                          json.dumps({
                              'compose': compose_obj.compose_id,
                              'num_linked_images': imported_images,
                          }))
def release__import_from_composeinfo(request, composeinfo_json):
    """
    Import release including variants and architectures from composeinfo json.
    """
    ci = common_hacks.composeinfo_from_str(composeinfo_json)

    if ci.release.is_layered:
        base_product_obj, _ = _logged_get_or_create(
            request,
            models.BaseProduct,
            name=ci.base_product.name,
            short=ci.base_product.short.lower(),
            version=ci.base_product.version,
        )
    else:
        base_product_obj = None

    product_obj, _ = _logged_get_or_create(
        request, models.Product, name=ci.release.name, short=ci.release.short.lower()
    )
    product_version_obj, _ = _logged_get_or_create(
        request,
        models.ProductVersion,
        product=product_obj,
        name=ci.release.name,
        short=ci.release.short.lower(),
        version=ci.release.major_version,
    )

    release_type_obj = models.ReleaseType.objects.get(short=getattr(ci.release, "type", "ga"))
    release_obj, _ = _logged_get_or_create(
        request,
        models.Release,
        name=ci.release.name,
        short=ci.release.short.lower(),
        version=ci.release.version,
        base_product=base_product_obj,
        release_type=release_type_obj,
        product_version=product_version_obj,
    )

    # if not created:
    #    raise RuntimeError("Release already exists: %s" % release_obj)

    # We can't log variants immediately after they are created, as their export
    # includes architectures. Therefore they are collected in this list and
    # logged once import is done. This also nicely abstracts integrated
    # variants that may not be present.
    add_to_changelog = []

    for variant in ci.get_variants(recursive=True):
        variant_type = models.VariantType.objects.get(name=variant.type)
        release = variant.release
        integrated_variant = None
        if release.name:
            integrated_release = get_or_create_integrated_release(request, release_obj, release)
            integrated_variant, created = models.Variant.objects.get_or_create(
                release=integrated_release,
                variant_id=variant.id,
                variant_uid=variant.uid,
                variant_name=variant.name,
                variant_type=models.VariantType.objects.get(name="variant"),
            )
            if created:
                add_to_changelog.append(integrated_variant)
        variant_obj, created = models.Variant.objects.get_or_create(
            release=release_obj,
            variant_id=variant.id,
            variant_uid=variant.uid,
            variant_name=variant.name,
            variant_type=variant_type,
        )
        if created:
            add_to_changelog.append(variant_obj)
        for arch in variant.arches:
            arch_obj = common_models.Arch.objects.get(name=arch)
            var_arch_obj, _ = models.VariantArch.objects.get_or_create(arch=arch_obj, variant=variant_obj)
            if integrated_variant:
                models.VariantArch.objects.get_or_create(arch=arch_obj, variant=integrated_variant)

    for obj in add_to_changelog:
        _maybe_log(request, True, obj)
def release__import_from_composeinfo(request, composeinfo_json):
    """
    Import release including variants and architectures from composeinfo json.
    """
    ci = common_hacks.composeinfo_from_str(composeinfo_json)

    if ci.release.is_layered:
        base_product_obj, _ = models.BaseProduct.objects.get_or_create(
            name=ci.base_product.name,
            short=ci.base_product.short.lower(),
            version=ci.base_product.version
        )
    else:
        base_product_obj = None

    product_obj, _ = models.Product.objects.get_or_create(
        name=ci.release.name,
        short=ci.release.short.lower()
    )
    product_version_obj, _ = models.ProductVersion.objects.get_or_create(
        product=product_obj,
        name=ci.release.name,
        short=ci.release.short.lower(),
        version=ci.release.major_version
    )

    release_type_obj = models.ReleaseType.objects.get(short=getattr(ci.release, "type", "ga"))
    release_obj, created = models.Release.objects.get_or_create(
        name=ci.release.name,
        short=ci.release.short.lower(),
        version=ci.release.version,
        base_product=base_product_obj,
        release_type=release_type_obj,
        product_version=product_version_obj,
    )

    # if not created:
    #    raise RuntimeError("Release already exists: %s" % release_obj)

    for variant in ci.get_variants(recursive=True):
        variant_type = models.VariantType.objects.get(name=variant.type)
        release = variant.release
        integrated_variant = None
        if release.name:
            integrated_release = get_or_create_integrated_release(
                release_obj,
                release
            )
            integrated_variant, _ = models.Variant.objects.get_or_create(
                release=integrated_release,
                variant_id=variant.id,
                variant_uid=variant.uid,
                variant_name=variant.name,
                variant_type=models.VariantType.objects.get(name='variant')
            )
        variant_obj, _ = models.Variant.objects.get_or_create(
            release=release_obj,
            variant_id=variant.id,
            variant_uid=variant.uid,
            variant_name=variant.name,
            variant_type=variant_type,
        )
        for arch in variant.arches:
            arch_obj = common_models.Arch.objects.get(name=arch)
            var_arch_obj, _ = models.VariantArch.objects.get_or_create(
                arch=arch_obj,
                variant=variant_obj
            )
            if integrated_variant:
                models.VariantArch.objects.get_or_create(
                    arch=arch_obj,
                    variant=integrated_variant
                )
def compose__import_rpms(request, release_id, composeinfo, rpm_manifest):
    release_obj = release_models.Release.objects.get(release_id=release_id)

    ci = common_hacks.composeinfo_from_str(composeinfo)
    rm = common_hacks.rpms_from_str(rpm_manifest)

    _maybe_raise_inconsistency_error(ci, rm, 'rpms')

    compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6], ci.compose.date[6:])
    compose_type = models.ComposeType.objects.get(name=ci.compose.type)
    acceptance_status = models.ComposeAcceptanceTestingState.objects.get(name='untested')
    compose_obj, created = lib._logged_get_or_create(
        request, models.Compose,
        release=release_obj,
        compose_id=ci.compose.id,
        compose_date=compose_date,
        compose_type=compose_type,
        compose_respin=ci.compose.respin,
        compose_label=ci.compose.label or None,
        acceptance_testing=acceptance_status,
    )
    if created and hasattr(request._request, '_messagings'):
        # add message
        _add_compose_create_msg(request, compose_obj)

    rpms_in_db = {}
    qs = package_models.RPM.objects.all()
    for rpm in qs.iterator():
        key = "%s-%s:%s-%s.%s" % (rpm.name, rpm.epoch, rpm.version, rpm.release, rpm.arch)
        rpms_in_db[key] = rpm.id

    cursor = connection.cursor()
    add_to_changelog = []
    imported_rpms = 0

    for variant in ci.get_variants(recursive=True):
        _link_compose_to_integrated_product(request, compose_obj, variant)
        variant_type = release_models.VariantType.objects.get(name=variant.type)
        variant_obj, created = models.Variant.objects.get_or_create(
            compose=compose_obj,
            variant_id=variant.id,
            variant_uid=variant.uid,
            variant_name=variant.name,
            variant_type=variant_type
        )
        if created:
            add_to_changelog.append(variant_obj)
        for arch in variant.arches:
            arch_obj = common_models.Arch.objects.get(name=arch)
            var_arch_obj, _ = models.VariantArch.objects.get_or_create(arch=arch_obj,
                                                                       variant=variant_obj)

            compose_rpms_in_db = set()
            qs = models.ComposeRPM.objects.filter(variant_arch=var_arch_obj).values_list('variant_arch_id',
                                                                                         'rpm_id')
            for (variant_arch_id, rpm_id) in qs.iterator():
                key = "%s/%s" % (variant_arch_id, rpm_id)
                compose_rpms_in_db.add(key)

            sources = set()
            for srpm_nevra, rpms in rm.rpms.get(variant.uid, {}).get(arch, {}).iteritems():
                sources.add(srpm_nevra)
                for rpm_nevra, rpm_data in rpms.iteritems():
                    imported_rpms += 1
                    path, filename = os.path.split(rpm_data['path'])
                    rpm_id = get_or_insert_rpm(rpms_in_db, cursor, rpm_nevra, srpm_nevra, filename)
                    sigkey_id = common_models.SigKey.get_cached_id(rpm_data["sigkey"], create=True)
                    path_id = models.Path.get_cached_id(path, create=True)
                    content_category = rpm_data["category"]
                    content_category_id = repository_models.ContentCategory.get_cached_id(content_category)
                    insert_compose_rpms_if_nonexist(compose_rpms_in_db, cursor,
                                                    var_arch_obj.id, rpm_id,
                                                    content_category_id, sigkey_id, path_id)

    for obj in add_to_changelog:
        lib._maybe_log(request, True, obj)

    request.changeset.add('notice', 0, 'null',
                          json.dumps({
                              'compose': compose_obj.compose_id,
                              'num_linked_rpms': imported_rpms,
                          }))