def _store_relative_path_for_compose(compose_obj, variants_info, variant, variant_obj, add_to_changelog):
vp = productmd.composeinfo.VariantPaths(variant)
common_hacks.deserialize_wrapper(vp.deserialize, variants_info.get(variant.name, {}).get("paths", {}))
for path_type in vp._fields:
path_type_obj, created = models.PathType.objects.get_or_create(name=path_type)
if created:
add_to_changelog.append(path_type_obj)
for arch in variant.arches:
field_value = getattr(vp, path_type)
if field_value and field_value.get(arch, None):
arch_obj = common_models.Arch.objects.get(name=arch)
crp_obj, created = models.ComposeRelPath.objects.get_or_create(
arch=arch_obj, variant=variant_obj, compose=compose_obj, type=path_type_obj, path=field_value[arch]
)
if created:
add_to_changelog.append(crp_obj)
def _store_relative_path_for_compose(compose_obj, variants_info, variant, variant_obj, add_to_changelog):
vp = productmd.composeinfo.VariantPaths(variant)
common_hacks.deserialize_wrapper(vp.deserialize, variants_info.get(variant.name, {}).get('paths', {}))
for path_type in vp._fields:
path_type_obj, created = models.PathType.objects.get_or_create(name=path_type)
if created:
add_to_changelog.append(path_type_obj)
for arch in variant.arches:
field_value = getattr(vp, path_type)
if field_value and field_value.get(arch, None):
arch_obj = common_models.Arch.objects.get(name=arch)
crp_obj, created = models.ComposeRelPath.objects.get_or_create(arch=arch_obj, variant=variant_obj,
compose=compose_obj, type=path_type_obj,
path=field_value[arch])
if created:
add_to_changelog.append(crp_obj)
def _set_compose_tree_location(request, compose_id, composeinfo, location, url,
scheme):
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo)
num_set_locations = 0
synced_content = [item.name for item in ContentCategory.objects.all()]
for variant in ci.get_variants(recursive=True):
variant_uid = variant.uid
variant_obj = models.Variant.objects.get(
compose__compose_id=compose_id, variant_uid=variant_uid)
for arch_name in variant.arches:
data = {
'compose': compose_id,
'variant': variant_uid,
'arch': arch_name,
'location': location,
'url': url,
'scheme': scheme,
'synced_content': synced_content
}
request.data['compose'] = compose_id
try:
obj = models.ComposeTree.objects.get(
compose__compose_id=compose_id,
variant=variant_obj,
arch__name=arch_name,
location__short=location)
# update
serializer = ComposeTreeSerializer(
obj, data=data, many=False, context={'request': request})
except models.ComposeTree.DoesNotExist:
# create
serializer = ComposeTreeSerializer(
data=data, many=False, context={'request': request})
if serializer.is_valid(raise_exception=True):
serializer.save()
num_set_locations += 1
request.changeset.add(
'notice', 0, 'null',
json.dumps({
'compose': compose_id,
'num_set_locations': num_set_locations,
}))
return num_set_locations
def _set_compose_tree_location(request, compose_id, composeinfo, location, url, scheme):
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo)
num_set_locations = 0
synced_content = [item.name for item in ContentCategory.objects.all()]
for variant in ci.get_variants(recursive=True):
variant_uid = variant.uid
variant_obj = models.Variant.objects.get(compose__compose_id=compose_id, variant_uid=variant_uid)
for arch_name in variant.arches:
data = {
"compose": compose_id,
"variant": variant_uid,
"arch": arch_name,
"location": location,
"url": url,
"scheme": scheme,
"synced_content": synced_content,
}
request.data["compose"] = compose_id
try:
obj = models.ComposeTree.objects.get(
compose__compose_id=compose_id, variant=variant_obj, arch__name=arch_name, location__short=location
)
# update
serializer = ComposeTreeSerializer(obj, data=data, many=False, context={"request": request})
except models.ComposeTree.DoesNotExist:
# create
serializer = ComposeTreeSerializer(data=data, many=False, context={"request": request})
if serializer.is_valid(raise_exception=True):
serializer.save()
num_set_locations += 1
request.changeset.add(
"notice", 0, "null", json.dumps({"compose": compose_id, "num_set_locations": num_set_locations})
)
return num_set_locations
def _set_compose_tree_location(request, compose_id, composeinfo, location, url, scheme):
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo)
num_set_locations = 0
synced_content = [item.name for item in ContentCategory.objects.all()]
for variant in ci.get_variants(recursive=True):
variant_uid = variant.uid
variant_obj = models.Variant.objects.get(compose__compose_id=compose_id, variant_uid=variant_uid)
for arch_name in variant.arches:
data = {'compose': compose_id,
'variant': variant_uid,
'arch': arch_name,
'location': location,
'url': url,
'scheme': scheme,
'synced_content': synced_content}
request.data['compose'] = compose_id
try:
obj = models.ComposeTree.objects.get(compose__compose_id=compose_id, variant=variant_obj,
arch__name=arch_name, location__short=location)
# update
serializer = ComposeTreeSerializer(obj, data=data, many=False, context={'request': request})
except models.ComposeTree.DoesNotExist:
# create
serializer = ComposeTreeSerializer(data=data, many=False, context={'request': request})
if serializer.is_valid(raise_exception=True):
serializer.save()
num_set_locations += 1
request.changeset.add('notice', 0, 'null',
json.dumps({
'compose': compose_id,
'num_set_locations': num_set_locations,
}))
return num_set_locations
def compose__import_images(request, release_id, composeinfo, image_manifest):
release_obj = release_models.Release.objects.get(release_id=release_id)
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo)
im = productmd.images.Images()
common_hacks.deserialize_wrapper(im.deserialize, image_manifest)
_maybe_raise_inconsistency_error(ci, im, "images")
compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6], ci.compose.date[6:])
compose_type = models.ComposeType.objects.get(name=ci.compose.type)
compose_obj, created = lib._logged_get_or_create(
request,
models.Compose,
release=release_obj,
compose_id=ci.compose.id,
compose_date=compose_date,
compose_type=compose_type,
compose_respin=ci.compose.respin,
compose_label=ci.compose.label or None,
)
if created and hasattr(request._request, "_messagings"):
# add message
_add_compose_create_msg(request, compose_obj)
add_to_changelog = []
imported_images = 0
variants_info = composeinfo["payload"]["variants"]
for variant in ci.get_variants(recursive=True):
_link_compose_to_integrated_product(request, compose_obj, variant)
variant_type = release_models.VariantType.objects.get(name=variant.type)
variant_obj, created = models.Variant.objects.get_or_create(
compose=compose_obj,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=variant_type,
)
if created:
add_to_changelog.append(variant_obj)
_store_relative_path_for_compose(compose_obj, variants_info, variant, variant_obj, add_to_changelog)
for arch in variant.arches:
arch_obj = common_models.Arch.objects.get(name=arch)
var_arch_obj, created = models.VariantArch.objects.get_or_create(arch=arch_obj, variant=variant_obj)
for i in im.images.get(variant.uid, {}).get(arch, []):
path, file_name = os.path.split(i.path)
path_id = models.Path.get_cached_id(path, create=True)
image, _ = package_models.Image.objects.get_or_create(
file_name=file_name,
sha256=i.checksums["sha256"],
defaults={
"image_format_id": package_models.ImageFormat.get_cached_id(i.format),
"image_type_id": package_models.ImageType.get_cached_id(i.type),
"disc_number": i.disc_number,
"disc_count": i.disc_count,
"arch": i.arch,
"mtime": i.mtime,
"size": i.size,
"bootable": i.bootable,
"implant_md5": i.implant_md5,
"volume_id": i.volume_id,
"md5": i.checksums.get("md5", None),
"sha1": i.checksums.get("sha1", None),
"subvariant": getattr(i, "subvariant", None),
},
)
mi, created = models.ComposeImage.objects.get_or_create(
variant_arch=var_arch_obj, image=image, path_id=path_id
)
imported_images += 1
for obj in add_to_changelog:
lib._maybe_log(request, True, obj)
request.changeset.add(
"notice", 0, "null", json.dumps({"compose": compose_obj.compose_id, "num_linked_images": imported_images})
)
if hasattr(request._request, "_messagings"):
_add_import_msg(request, compose_obj, "images", imported_images)
return compose_obj.compose_id, imported_images
def compose__import_rpms(request, release_id, composeinfo, rpm_manifest):
release_obj = release_models.Release.objects.get(release_id=release_id)
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo)
rm = Rpms()
common_hacks.deserialize_wrapper(rm.deserialize, rpm_manifest)
_maybe_raise_inconsistency_error(ci, rm, "rpms")
compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6], ci.compose.date[6:])
compose_type = models.ComposeType.objects.get(name=ci.compose.type)
acceptance_status = models.ComposeAcceptanceTestingState.objects.get(name="untested")
compose_obj, created = lib._logged_get_or_create(
request,
models.Compose,
release=release_obj,
compose_id=ci.compose.id,
compose_date=compose_date,
compose_type=compose_type,
compose_respin=ci.compose.respin,
compose_label=ci.compose.label or None,
acceptance_testing=acceptance_status,
)
if created and hasattr(request._request, "_messagings"):
# add message
_add_compose_create_msg(request, compose_obj)
rpms_in_db = {}
qs = package_models.RPM.objects.all()
for rpm in qs.iterator():
key = "%s-%s:%s-%s.%s" % (rpm.name, rpm.epoch, rpm.version, rpm.release, rpm.arch)
rpms_in_db[key] = rpm.id
cursor = connection.cursor()
add_to_changelog = []
imported_rpms = 0
variants_info = composeinfo["payload"]["variants"]
for variant in ci.get_variants(recursive=True):
_link_compose_to_integrated_product(request, compose_obj, variant)
variant_type = release_models.VariantType.objects.get(name=variant.type)
variant_obj, created = models.Variant.objects.get_or_create(
compose=compose_obj,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=variant_type,
)
if created:
add_to_changelog.append(variant_obj)
_store_relative_path_for_compose(compose_obj, variants_info, variant, variant_obj, add_to_changelog)
for arch in variant.arches:
arch_obj = common_models.Arch.objects.get(name=arch)
var_arch_obj, _ = models.VariantArch.objects.get_or_create(arch=arch_obj, variant=variant_obj)
compose_rpms_in_db = set()
qs = models.ComposeRPM.objects.filter(variant_arch=var_arch_obj).values_list("variant_arch_id", "rpm_id")
for (variant_arch_id, rpm_id) in qs.iterator():
key = "%s/%s" % (variant_arch_id, rpm_id)
compose_rpms_in_db.add(key)
sources = set()
for srpm_nevra, rpms in rm.rpms.get(variant.uid, {}).get(arch, {}).iteritems():
sources.add(srpm_nevra)
for rpm_nevra, rpm_data in rpms.iteritems():
imported_rpms += 1
path, filename = os.path.split(rpm_data["path"])
rpm_id = get_or_insert_rpm(rpms_in_db, cursor, rpm_nevra, srpm_nevra, filename)
sigkey_id = common_models.SigKey.get_cached_id(rpm_data["sigkey"], create=True)
path_id = models.Path.get_cached_id(path, create=True)
content_category = rpm_data["category"]
content_category_id = repository_models.ContentCategory.get_cached_id(content_category)
insert_compose_rpms_if_nonexist(
compose_rpms_in_db, cursor, var_arch_obj.id, rpm_id, content_category_id, sigkey_id, path_id
)
for obj in add_to_changelog:
lib._maybe_log(request, True, obj)
request.changeset.add(
"notice", 0, "null", json.dumps({"compose": compose_obj.compose_id, "num_linked_rpms": imported_rpms})
)
if hasattr(request._request, "_messagings"):
_add_import_msg(request, compose_obj, "rpms", imported_rpms)
return compose_obj.compose_id, imported_rpms
def release__import_from_composeinfo(request, composeinfo_json):
"""
Import release including variants and architectures from composeinfo json.
"""
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo_json)
if ci.release.is_layered:
base_product_obj, _ = _logged_get_or_create(
request, models.BaseProduct,
name=ci.base_product.name,
short=ci.base_product.short.lower(),
version=ci.base_product.version
)
else:
base_product_obj = None
product_obj, _ = _logged_get_or_create(
request, models.Product,
name=ci.release.name,
short=ci.release.short.lower()
)
product_version_obj, _ = _logged_get_or_create(
request, models.ProductVersion,
product=product_obj,
name=ci.release.name,
short=ci.release.short.lower(),
version=ci.release.major_version
)
release_type_obj = models.ReleaseType.objects.get(short=getattr(ci.release, "type", "ga"))
release_obj, _ = _logged_get_or_create(
request, models.Release,
name=ci.release.name,
short=ci.release.short.lower(),
version=ci.release.version,
base_product=base_product_obj,
release_type=release_type_obj,
product_version=product_version_obj,
)
# if not created:
# raise RuntimeError("Release already exists: %s" % release_obj)
# We can't log variants immediately after they are created, as their export
# includes architectures. Therefore they are collected in this list and
# logged once import is done. This also nicely abstracts integrated
# variants that may not be present.
add_to_changelog = []
for variant in ci.variants.get_variants(recursive=True):
variant_type = models.VariantType.objects.get(name=variant.type)
release = variant.release
integrated_variant = None
if release.name:
integrated_release = get_or_create_integrated_release(
request,
release_obj,
release
)
integrated_variant, created = models.Variant.objects.get_or_create(
release=integrated_release,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=models.VariantType.objects.get(name='variant')
)
if created:
add_to_changelog.append(integrated_variant)
variant_obj, created = models.Variant.objects.get_or_create(
release=release_obj,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=variant_type,
)
if created:
add_to_changelog.append(variant_obj)
for arch in variant.arches:
arch_obj = common_models.Arch.objects.get(name=arch)
var_arch_obj, _ = models.VariantArch.objects.get_or_create(
arch=arch_obj,
variant=variant_obj
)
if integrated_variant:
models.VariantArch.objects.get_or_create(
arch=arch_obj,
variant=integrated_variant
)
for obj in add_to_changelog:
_maybe_log(request, True, obj)
return release_obj
def release__import_from_composeinfo(request, composeinfo_json):
"""
Import release including variants and architectures from composeinfo json.
"""
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo_json)
if ci.release.is_layered:
base_product_obj, _ = _logged_get_or_create(
request,
models.BaseProduct,
name=ci.base_product.name,
short=ci.base_product.short.lower(),
version=ci.base_product.version)
else:
base_product_obj = None
product_obj, _ = _logged_get_or_create(request,
models.Product,
name=ci.release.name,
short=ci.release.short.lower())
product_version_obj, _ = _logged_get_or_create(
request,
models.ProductVersion,
product=product_obj,
name=ci.release.name,
short=ci.release.short.lower(),
version=ci.release.major_version)
release_type_obj = models.ReleaseType.objects.get(
short=getattr(ci.release, "type", "ga"))
release_obj, _ = _logged_get_or_create(
request,
models.Release,
name=ci.release.name,
short=ci.release.short.lower(),
version=ci.release.version,
base_product=base_product_obj,
release_type=release_type_obj,
product_version=product_version_obj,
)
# if not created:
# raise RuntimeError("Release already exists: %s" % release_obj)
# We can't log variants immediately after they are created, as their export
# includes architectures. Therefore they are collected in this list and
# logged once import is done. This also nicely abstracts integrated
# variants that may not be present.
add_to_changelog = []
for variant in ci.variants.get_variants(recursive=True):
variant_type = models.VariantType.objects.get(name=variant.type)
release = variant.release
integrated_variant = None
if release.name:
integrated_release = get_or_create_integrated_release(
request, release_obj, release)
integrated_variant, created = models.Variant.objects.get_or_create(
release=integrated_release,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=models.VariantType.objects.get(name='variant'))
if created:
add_to_changelog.append(integrated_variant)
variant_obj, created = models.Variant.objects.get_or_create(
release=release_obj,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=variant_type,
)
if created:
add_to_changelog.append(variant_obj)
for arch in variant.arches:
arch_obj = common_models.Arch.objects.get(name=arch)
var_arch_obj, _ = models.VariantArch.objects.get_or_create(
arch=arch_obj, variant=variant_obj)
if integrated_variant:
models.VariantArch.objects.get_or_create(
arch=arch_obj, variant=integrated_variant)
for obj in add_to_changelog:
_maybe_log(request, True, obj)
return release_obj
def compose__import_rpms(request, release_id, composeinfo, rpm_manifest):
release_obj = release_models.Release.objects.get(release_id=release_id)
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo)
rm = productmd.rpms.Rpms()
common_hacks.deserialize_wrapper(rm.deserialize, rpm_manifest)
_maybe_raise_inconsistency_error(ci, rm, 'rpms')
compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6],
ci.compose.date[6:])
compose_type = models.ComposeType.objects.get(name=ci.compose.type)
acceptance_status = models.ComposeAcceptanceTestingState.objects.get(
name='untested')
compose_obj, created = lib._logged_get_or_create(
request,
models.Compose,
release=release_obj,
compose_id=ci.compose.id,
compose_date=compose_date,
compose_type=compose_type,
compose_respin=ci.compose.respin,
compose_label=ci.compose.label or None,
acceptance_testing=acceptance_status,
)
if created and hasattr(request._request, '_messagings'):
# add message
_add_compose_create_msg(request, compose_obj)
rpms_in_db = {}
qs = package_models.RPM.objects.all()
for rpm in qs.iterator():
key = "%s-%s:%s-%s.%s" % (rpm.name, rpm.epoch, rpm.version,
rpm.release, rpm.arch)
rpms_in_db[key] = rpm.id
cursor = connection.cursor()
add_to_changelog = []
imported_rpms = 0
for variant in ci.get_variants(recursive=True):
_link_compose_to_integrated_product(request, compose_obj, variant)
variant_type = release_models.VariantType.objects.get(
name=variant.type)
variant_obj, created = models.Variant.objects.get_or_create(
compose=compose_obj,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=variant_type)
if created:
add_to_changelog.append(variant_obj)
for arch in variant.arches:
arch_obj = common_models.Arch.objects.get(name=arch)
var_arch_obj, _ = models.VariantArch.objects.get_or_create(
arch=arch_obj, variant=variant_obj)
compose_rpms_in_db = set()
qs = models.ComposeRPM.objects.filter(
variant_arch=var_arch_obj).values_list('variant_arch_id',
'rpm_id')
for (variant_arch_id, rpm_id) in qs.iterator():
key = "%s/%s" % (variant_arch_id, rpm_id)
compose_rpms_in_db.add(key)
sources = set()
for srpm_nevra, rpms in rm.rpms.get(variant.uid,
{}).get(arch, {}).iteritems():
sources.add(srpm_nevra)
for rpm_nevra, rpm_data in rpms.iteritems():
imported_rpms += 1
path, filename = os.path.split(rpm_data['path'])
rpm_id = get_or_insert_rpm(rpms_in_db, cursor, rpm_nevra,
srpm_nevra, filename)
sigkey_id = common_models.SigKey.get_cached_id(
rpm_data["sigkey"], create=True)
path_id = models.Path.get_cached_id(path, create=True)
content_category = rpm_data["category"]
content_category_id = repository_models.ContentCategory.get_cached_id(
content_category)
insert_compose_rpms_if_nonexist(compose_rpms_in_db, cursor,
var_arch_obj.id, rpm_id,
content_category_id,
sigkey_id, path_id)
for obj in add_to_changelog:
lib._maybe_log(request, True, obj)
request.changeset.add(
'notice', 0, 'null',
json.dumps({
'compose': compose_obj.compose_id,
'num_linked_rpms': imported_rpms,
}))
return compose_obj.compose_id, imported_rpms
def compose__import_images(request, release_id, composeinfo, image_manifest):
release_obj = release_models.Release.objects.get(release_id=release_id)
ci = productmd.composeinfo.ComposeInfo()
common_hacks.deserialize_wrapper(ci.deserialize, composeinfo)
im = productmd.images.Images()
common_hacks.deserialize_wrapper(im.deserialize, image_manifest)
_maybe_raise_inconsistency_error(ci, im, 'images')
compose_date = "%s-%s-%s" % (ci.compose.date[:4], ci.compose.date[4:6],
ci.compose.date[6:])
compose_type = models.ComposeType.objects.get(name=ci.compose.type)
compose_obj, created = lib._logged_get_or_create(
request,
models.Compose,
release=release_obj,
compose_id=ci.compose.id,
compose_date=compose_date,
compose_type=compose_type,
compose_respin=ci.compose.respin,
compose_label=ci.compose.label or None,
)
if created and hasattr(request._request, '_messagings'):
# add message
_add_compose_create_msg(request, compose_obj)
add_to_changelog = []
imported_images = 0
for variant in ci.get_variants(recursive=True):
_link_compose_to_integrated_product(request, compose_obj, variant)
variant_type = release_models.VariantType.objects.get(
name=variant.type)
variant_obj, created = models.Variant.objects.get_or_create(
compose=compose_obj,
variant_id=variant.id,
variant_uid=variant.uid,
variant_name=variant.name,
variant_type=variant_type)
if created:
add_to_changelog.append(variant_obj)
for arch in variant.arches:
arch_obj = common_models.Arch.objects.get(name=arch)
var_arch_obj, created = models.VariantArch.objects.get_or_create(
arch=arch_obj, variant=variant_obj)
for i in im.images.get(variant.uid, {}).get(arch, []):
path, file_name = os.path.split(i.path)
path_id = models.Path.get_cached_id(path, create=True)
image, _ = package_models.Image.objects.get_or_create(
file_name=file_name,
sha256=i.checksums["sha256"],
defaults={
'image_format_id':
package_models.ImageFormat.get_cached_id(i.format),
'image_type_id':
package_models.ImageType.get_cached_id(i.type),
'disc_number':
i.disc_number,
'disc_count':
i.disc_count,
'arch':
i.arch,
'mtime':
i.mtime,
'size':
i.size,
'bootable':
i.bootable,
'implant_md5':
i.implant_md5,
'volume_id':
i.volume_id,
'md5':
i.checksums.get("md5", None),
'sha1':
i.checksums.get("sha1", None),
})
mi, created = models.ComposeImage.objects.get_or_create(
variant_arch=var_arch_obj, image=image, path_id=path_id)
imported_images += 1
for obj in add_to_changelog:
lib._maybe_log(request, True, obj)
request.changeset.add(
'notice', 0, 'null',
json.dumps({
'compose': compose_obj.compose_id,
'num_linked_images': imported_images,
}))
return compose_obj.compose_id, imported_images