async def run(self):
"""
Build and emit `DeclarativeContent` from the Manifest data.
If a cookbook specifier is set in the remote, cookbooks are filtered
using this specifier.
"""
with ProgressBar(message="Downloading Metadata", total=1) as pb:
downloader = self.remote.get_downloader(url=urljoin(self.remote.url + "/", "universe"))
result = await downloader.run()
pb.increment()
cookbook_names = self.remote.specifier_cookbook_names()
with ProgressBar(message="Parsing Metadata") as pb:
universe = Universe(result.path)
for entry in universe.read():
if cookbook_names and entry.name not in cookbook_names:
continue
cookbook = CookbookPackageContent(
name=entry.name, version=entry.version, dependencies=entry.dependencies
)
artifact = Artifact()
da = DeclarativeArtifact(
artifact=artifact,
url=entry.download_url,
relative_path=cookbook.relative_path(),
remote=self.remote,
deferred_download=not self.download_artifacts,
)
dc = DeclarativeContent(content=cookbook, d_artifacts=[da])
pb.increment()
await self.put(dc)
def test_content_associated_using_repo_key(self):
stage = QueryExistingRepoContentAndArtifacts(
new_version=self.new_version_all_content())
# c1: Existing content unit with Artifact
c1 = CookbookPackageContent(name="c1",
version="1.0.0",
dependencies={})
# c2: content unit does not exist in DB
c2 = CookbookPackageContent(name="c2",
version="1.0.0",
dependencies={})
# c3: content unit does exist, has a content_artifact association,
# but no artifact (i.e. is a non-immediate content unit)
c3 = CookbookPackageContent(name="c3",
version="1.0.0",
dependencies={})
d_c1_d_a1 = DeclarativeArtifact(
artifact=Artifact(),
url="http://a1",
relative_path=c1.relative_path(),
remote=self.remote,
)
d_c2_d_a2 = DeclarativeArtifact(
artifact=Artifact(),
url="http://a2",
relative_path=c2.relative_path(),
remote=self.remote,
)
d_c3_d_a3 = DeclarativeArtifact(
artifact=Artifact(),
url="http://a3",
relative_path=c3.relative_path(),
remote=self.remote,
)
batch = [
DeclarativeContent(content=c1, d_artifacts=[d_c1_d_a1]),
DeclarativeContent(content=c2, d_artifacts=[d_c2_d_a2]),
DeclarativeContent(content=c3, d_artifacts=[d_c3_d_a3]),
]
stage._process_batch(batch)
self.assertEqual(batch[0].content.content_id, "1")
self.assertEqual(batch[0].content.pk, self.c1.pk)
self.assertEqual(batch[0].d_artifacts[0].artifact.pk, self.a1.pk)
self.assertIsNone(batch[1].content.pk)
self.assertTrue(batch[1].d_artifacts[0].artifact._state.adding)
self.assertEqual(batch[2].content.pk, self.c3.pk)
self.assertTrue(batch[2].d_artifacts[0].artifact._state.adding)
def test_content_associated_using_repo_key(self):
dc_c1 = CookbookPackageContent(name="c1",
version="1.0.0",
dependencies={})
dc_c2 = CookbookPackageContent(name="c2",
version="1.0.0",
dependencies={})
batch = [
DeclarativeContent(content=dc_c1, d_artifacts=[]),
DeclarativeContent(content=dc_c2, d_artifacts=[]),
]
stage = QueryExistingContentUnits(
new_version=self.new_version_all_content())
stage._process_batch(batch)
self.assertEqual(batch[0].content.content_id, "1")
self.assertEqual(batch[0].content.pk, self.c1.pk)
self.assertIsNone(batch[1].content.pk)
def test_content_associated_using_natural_key(self):
dc_c1 = CookbookPackageContent(name="c1",
version="1.0.0",
content_id="1",
dependencies={})
dc_c1_other = CookbookPackageContent(name="c1",
version="1.0.0",
content_id="other",
dependencies={})
batch = [
DeclarativeContent(content=dc_c1, d_artifacts=[]),
DeclarativeContent(content=dc_c1_other, d_artifacts=[]),
]
stage = QueryExistingContentUnits()
stage._process_batch(batch)
self.assertEqual(batch[0].content.content_id, "1")
self.assertEqual(batch[0].content.pk, self.c1.pk)
self.assertIsNone(batch[1].content.pk)
async def __call__(self, in_q, out_q):
"""
Build and emit `DeclarativeContent` from the Manifest data.
If a cookbook specifier is set in the remote, cookbooks are filtered
using this specifier.
Args: in_q (asyncio.Queue): Unused because the first stage doesn't read
from an input queue. out_q (asyncio.Queue): The out_q to send
`DeclarativeContent` objects to
"""
with ProgressBar(message='Downloading Metadata', total=1) as pb:
downloader = self.remote.get_downloader(
url=urljoin(self.remote.url + '/', 'universe'))
result = await downloader.run()
pb.increment()
cookbook_names = self.remote.specifier_cookbook_names()
with ProgressBar(message='Parsing Metadata') as pb:
universe = Universe(result.path)
for entry in universe.read():
if cookbook_names and entry.name not in cookbook_names:
continue
cookbook = CookbookPackageContent(
name=entry.name,
version=entry.version,
dependencies=entry.dependencies)
artifact = Artifact()
da = DeclarativeArtifact(artifact, entry.download_url,
cookbook.relative_path(), self.remote)
dc = DeclarativeContent(content=cookbook, d_artifacts=[da])
pb.increment()
await out_q.put(dc)
await out_q.put(None)
def check_repo_version_constraint(publication):
"""
Ensure that repo version to publish fulfills repo_key_fields() uniqueness.
Raises:
ValueError: When constraint is violated
"""
fields = CookbookPackageContent.repo_key_fields()
qs_content = CookbookPackageContent.objects.filter(
pk__in=publication.repository_version.content)
qs = qs_content.values(*fields).annotate(num_cookbooks=Count("pk")).filter(
num_cookbooks__gt=1)
duplicates = [f"{res['name']} {res['version']}" for res in qs]
if duplicates:
raise ValueError(
f"Publication would contain multiple versions of cookbooks: {', '.join(duplicates)}"
)
def test_existing_content_is_ok(self):
# dc_c1 is a duplicate of c1 existing in the DB
dc_c1 = CookbookPackageContent(name="c1",
version="1.0.0",
content_id="1",
dependencies={})
c2 = CookbookPackageContent.objects.create(name="c2",
version="1.0.0",
content_id="2",
dependencies={})
batch = [
DeclarativeContent(content=dc_c1, d_artifacts=[]),
DeclarativeContent(content=c2, d_artifacts=[]),
]
stage = QueryExistingContentUnits()
stage._process_batch(batch)
self.assertEqual(batch[0].content.pk, self.c1.pk)
self.assertEqual(batch[1].content.pk, c2.pk)
def deferred_validate(self, data):
"""Validate that the artifact is a cookbook and extract it's meta-data."""
data = super().deferred_validate(data)
try:
metadata = CookbookMetadata.from_cookbook_file(
fileobj=data["artifact"].file, name=data["name"])
except FileNotFoundError:
raise serializers.ValidationError(
detail={
"artifact": _("No metadata.json found in cookbook tar")
})
try:
if data["version"] != metadata.version:
raise serializers.ValidationError(
detail={
"version":
_("version does not correspond to version in cookbook tar"
)
})
except KeyError:
pass
data["version"] = metadata.version
data["dependencies"] = metadata.dependencies
data["content_id_type"] = self.Meta.model.SHA256
data["content_id"] = data["artifact"].sha256
data["relative_path"] = CookbookPackageContent.relative_path_from_data(
data)
content = self.Meta.model.objects.filter(
content_id_type=data["content_id_type"],
content_id=data["content_id"],
name=data["name"],
version=data["version"],
)
if content.exists():
raise serializers.ValidationError(
_("There is already a cookbook package '{name}-{version}'"
" with sha256 '{content_id}'.").format(**data))
return data