async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None,
downloaded=True):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
For non-downloaded content, artifact with its expected checksum and size is created.
"""
if not downloaded:
if not expected_digests:
raise ValueError(
_('No digest is provided for on_demand content creation. Pulp 2 '
'storage path: {}'.format(pulp2_storage_path)))
artifact = Artifact(**expected_digests)
artifact.size = expected_size
return artifact
artifact = Artifact.init_and_validate(
pulp2_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
pulp3_storage_relative_path = storage.get_artifact_path(
artifact.sha256)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug(
_('Hard link cannot be created, file will be copied.'))
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
if not expected_digests:
expected_digests = {'sha256': artifact.sha256}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
def setUp(self):
with open(self.artifact01_path, 'w') as f:
f.write('Temp Artifact File 01')
with open(self.artifact02_path, 'w') as f:
f.write('Temp Artifact File 02')
self.artifact01 = Artifact.init_and_validate(self.artifact01_path)
self.artifact01.save()
self.artifact02 = Artifact.init_and_validate(self.artifact02_path)
self.artifact02.save()
def setUp(self):
with open(self.artifact01_path, "w") as f:
f.write("Temp Artifact File 01")
with open(self.artifact02_path, "w") as f:
f.write("Temp Artifact File 02")
self.artifact01 = Artifact.init_and_validate(self.artifact01_path)
self.artifact01.save()
self.artifact02 = Artifact.init_and_validate(self.artifact02_path)
self.artifact02.save()
def get_or_create_blob(layer_json, manifest, path):
"""
Creates Blob from json snippet of manifest.json
Args:
layer_json (json): json
manifest (class:`pulp_container.app.models.Manifest`): The manifest
path (str): Path of the directory that contains layer
Returns:
class:`pulp_container.app.models.Blob`
"""
try:
blob = Blob.objects.get(digest=layer_json["digest"])
except Blob.DoesNotExist:
layer_file_name = "{}{}".format(path, layer_json["digest"][7:])
layer_artifact = Artifact.init_and_validate(layer_file_name)
layer_artifact.save()
blob = Blob(digest=layer_json["digest"],
media_type=layer_json["mediaType"])
blob.save()
ContentArtifact(artifact=layer_artifact,
content=blob,
relative_path=layer_json["digest"]).save()
if blob.media_type != MEDIA_TYPE.CONFIG_BLOB_OCI:
BlobManifest(manifest=manifest, manifest_blob=blob).save()
return blob
def create(self, request):
"""
Dispatch a Collection creation task.
"""
serializer = CollectionOneShotSerializer(data=request.data, context={'request': request})
serializer.is_valid(raise_exception=True)
expected_digests = {}
if serializer.validated_data['sha256']:
expected_digests['sha256'] = serializer.validated_data['sha256']
try:
artifact = Artifact.init_and_validate(serializer.validated_data['file'],
expected_digests=expected_digests)
except DigestValidationError:
raise serializers.ValidationError(
_("The provided sha256 value does not match the sha256 of the uploaded file.")
)
artifact.save()
async_result = enqueue_with_reservation(
import_collection, [str(artifact.pk)],
kwargs={
'artifact_pk': artifact.pk,
}
)
return OperationPostponedResponse(async_result, request)
def post(self, request):
"""Upload an RPM package."""
serializer = OneShotUploadSerializer(
data=request.data, context={'request': request})
serializer.is_valid(raise_exception=True)
artifact = Artifact.init_and_validate(request.data['file'])
if 'repository' in request.data:
repository = serializer.validated_data['repository']
else:
repository = None
try:
artifact.save()
except IntegrityError:
# if artifact already exists, let's use it
artifact = Artifact.objects.get(sha256=artifact.sha256)
async_result = enqueue_with_reservation(
one_shot_upload, [artifact],
kwargs={
'artifact': artifact,
'repository': repository,
})
return OperationPostponedResponse(async_result, request)
def create(self, request):
"""Upload an RPM package."""
artifact = Artifact.init_and_validate(request.data['file'])
filename = request.data['file'].name
if 'repository' in request.data:
serializer = OneShotUploadSerializer(data=request.data,
context={'request': request})
serializer.is_valid(raise_exception=True)
repository = serializer.validated_data['repository']
repository_pk = repository.pk
else:
repository_pk = None
try:
artifact.save()
except IntegrityError:
# if artifact already exists, let's use it
artifact = Artifact.objects.get(sha256=artifact.sha256)
async_result = enqueue_with_reservation(tasks.one_shot_upload,
[artifact],
kwargs={
'artifact_pk': artifact.pk,
'filename': filename,
'repository_pk':
repository_pk,
})
return OperationPostponedResponse(async_result, request)
def validate(self, data):
"""Validates the request."""
action = data.get(":action")
if action != "file_upload":
raise serializers.ValidationError(
_("We do not support the :action {}").format(action))
file = data.get("content")
for ext, packagetype in DIST_EXTENSIONS.items():
if file.name.endswith(ext):
break
else:
raise serializers.ValidationError(
_("Extension on {} is not a valid python extension "
"(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(
file.name))
sha256 = data.get("sha256_digest")
digests = {"sha256": sha256} if sha256 else None
artifact = Artifact.init_and_validate(file, expected_digests=digests)
try:
artifact.save()
except IntegrityError:
artifact = Artifact.objects.get(sha256=artifact.sha256)
artifact.touch()
log.info(f"Artifact for {file.name} already existed in database")
data["content"] = (artifact, file.name)
return data
def validate(self, data):
"""Validates that all the fields make sense."""
data = super().validate(data)
if "containerfile" in data:
if "containerfile_artifact" in data:
raise serializers.ValidationError(
_("Only one of 'containerfile' and 'containerfile_artifact' may be specified.")
)
data["containerfile_artifact"] = Artifact.init_and_validate(data.pop("containerfile"))
elif "containerfile_artifact" not in data:
raise serializers.ValidationError(_("'containerfile' or 'containerfile_artifact' must "
"be specified."))
artifacts = {}
if 'artifacts' in data:
for url, relative_path in data['artifacts'].items():
if os.path.isabs(relative_path):
raise serializers.ValidationError(_("Relative path cannot start with '/'. "
"{0}").format(relative_path))
artifactfield = RelatedField(view_name='artifacts-detail',
queryset=Artifact.objects.all(),
source='*', initial=url)
try:
artifact = artifactfield.run_validation(data=url)
artifacts[artifact.pk] = relative_path
except serializers.ValidationError as e:
# Append the URL of missing Artifact to the error message
e.detail[0] = "%s %s" % (e.detail[0], url)
raise e
data['artifacts'] = artifacts
return data
def create(self, request):
"""
Dispatch a Collection creation task.
"""
serializer = CollectionOneShotSerializer(data=request.data,
context={"request": request})
serializer.is_valid(raise_exception=True)
expected_digests = {}
if serializer.validated_data["sha256"]:
expected_digests["sha256"] = serializer.validated_data["sha256"]
try:
artifact = Artifact.init_and_validate(
serializer.validated_data["file"],
expected_digests=expected_digests)
except DigestValidationError:
raise serializers.ValidationError(
_("The provided sha256 value does not match the sha256 of the uploaded file."
))
try:
artifact.save()
except IntegrityError:
raise serializers.ValidationError(_("Artifact already exists."))
async_result = self._dispatch_import_collection_task(artifact.pk)
return OperationPostponedResponse(async_result, request)
async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
"""
if not expected_digests.get('sha256'):
# TODO: all checksums are calculated for the pulp 2 storage path, is it ok?
artifact = Artifact.init_and_validate(pulp2_storage_path,
size=expected_size)
sha256digest = expected_digests.get('sha256') or artifact.sha256
pulp3_storage_relative_path = storage.get_artifact_path(sha256digest)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug('Hard link cannot be created, file will be copied.')
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
expected_digests = {'sha256': sha256digest}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
def create(self, request, path):
"""
Dispatch a Collection creation task.
"""
distro = get_object_or_404(AnsibleDistribution, base_path=path)
serializer = CollectionOneShotSerializer(data=request.data,
context={"request": request})
serializer.is_valid(raise_exception=True)
expected_digests = {}
if serializer.validated_data["sha256"]:
expected_digests["sha256"] = serializer.validated_data["sha256"]
try:
artifact = Artifact.init_and_validate(
serializer.validated_data["file"],
expected_digests=expected_digests)
except DigestValidationError:
raise serializers.ValidationError(
_("The provided sha256 value does not match the sha256 of the uploaded file."
))
try:
artifact.save()
except IntegrityError:
raise serializers.ValidationError(_("Artifact already exists."))
kwargs = {}
if serializer.validated_data["expected_namespace"]:
kwargs["expected_namespace"] = serializer.validated_data[
"expected_namespace"]
if serializer.validated_data["expected_name"]:
kwargs["expected_name"] = serializer.validated_data[
"expected_name"]
if serializer.validated_data["expected_version"]:
kwargs["expected_version"] = serializer.validated_data[
"expected_version"]
async_result = self._dispatch_import_collection_task(
artifact.pk, distro.repository, **kwargs)
CollectionImport.objects.create(task_id=async_result.id)
data = {
"task":
reverse(
"collection-imports-detail",
kwargs={
"path": path,
"pk": async_result.id
},
request=None,
)
}
return Response(data, status=http_status.HTTP_202_ACCEPTED)
def validate(self, data):
"""Validate the GemContent data."""
data = super().validate(data)
if "file" in data:
if "artifact" in data:
raise ValidationError(_("Only one of 'file' and 'artifact' may be specified."))
data["artifact"] = Artifact.init_and_validate(data.pop("file"))
elif "artifact" not in data:
raise ValidationError(_("One of 'file' and 'artifact' must be specified."))
if "request" not in self.context:
data = self.deferred_validate(data)
return data
def post(self, request, path):
"""
Queues a task that creates a new Collection from an uploaded artifact.
"""
distro = get_object_or_404(AnsibleDistribution, base_path=path)
serializer = GalaxyCollectionUploadSerializer(
data=request.data, context={"request": request})
serializer.is_valid(raise_exception=True)
artifact = Artifact.init_and_validate(
serializer.validated_data["file"])
artifact.save()
async_result = self._dispatch_import_collection_task(
artifact.pk, distro.repository)
return OperationPostponedResponse(async_result, request)
def _create_snippet(snippet_string):
"""
Create snippet of modulemd[-defaults] as artifact.
Args:
snippet_string (string):
Snippet with modulemd[-defaults] yaml
Returns:
Snippet as unsaved Artifact object
"""
tmp_file = tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False)
with open(tmp_file.name, "w") as snippet:
snippet.write(snippet_string)
return Artifact.init_and_validate(tmp_file.name)
def put(self, request, path, pk=None):
"""
Create a blob from uploaded chunks.
"""
_, repository = self.get_dr_push(request, path)
digest = request.query_params["digest"]
upload = models.Upload.objects.get(pk=pk, repository=repository)
chunks = UploadChunk.objects.filter(upload=upload).order_by("offset")
with NamedTemporaryFile("ab") as temp_file:
for chunk in chunks:
temp_file.write(chunk.file.read())
temp_file.flush()
uploaded_file = PulpTemporaryUploadedFile.from_file(
File(open(temp_file.name, "rb")))
if uploaded_file.hashers["sha256"].hexdigest() == digest[len("sha256:"
):]:
try:
artifact = Artifact.init_and_validate(uploaded_file)
artifact.save()
except IntegrityError:
artifact = Artifact.objects.get(sha256=artifact.sha256)
try:
blob = models.Blob(digest=digest,
media_type=models.MEDIA_TYPE.REGULAR_BLOB)
blob.save()
except IntegrityError:
blob = models.Blob.objects.get(digest=digest)
try:
blob_artifact = ContentArtifact(artifact=artifact,
content=blob,
relative_path=digest)
blob_artifact.save()
except IntegrityError:
pass
with repository.new_version() as new_version:
new_version.add_content(models.Blob.objects.filter(pk=blob.pk))
upload.delete()
return BlobResponse(blob, path, 201, request)
else:
raise Exception("The digest did not match")
def setUp(self):
with open(self.artifact_path, 'w') as f:
f.write('Temp Artifact File')
self.artifact = Artifact.init_and_validate(self.artifact_path)
self.artifact.save()
collection = Collection.objects.create(namespace='my_ns',
name='my_name')
self.collection_version = CollectionVersion.objects.create(
collection=collection)
self.collection_version.save()
content_artifact = ContentArtifact.objects.create(
artifact=self.artifact,
content=self.collection_version,
)
content_artifact.save()
def add_image_from_directory_to_repository(path, repository, tag):
"""
Creates a Manifest and all blobs from a directory with OCI image
Args:
path (str): Path to directory with the OCI image
repository (class:`pulpcore.plugin.models.Repository`): The destination repository
tag (str): Tag name for the new image in the repository
Returns:
A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container
image and tag.
"""
manifest_path = "{}manifest.json".format(path)
manifest_artifact = Artifact.init_and_validate(manifest_path)
manifest_artifact.save()
manifest_digest = "sha256:{}".format(manifest_artifact.sha256)
manifest = Manifest(digest=manifest_digest,
schema_version=2,
media_type=MEDIA_TYPE.MANIFEST_OCI)
manifest.save()
ContentArtifact(artifact=manifest_artifact,
content=manifest,
relative_path=manifest_digest).save()
tag = Tag(name=tag, tagged_manifest=manifest)
tag.save()
ContentArtifact(artifact=manifest_artifact,
content=tag,
relative_path=tag.name).save()
with repository.new_version() as new_repo_version:
new_repo_version.add_content(Manifest.objects.filter(pk=manifest.pk))
new_repo_version.add_content(Tag.objects.filter(pk=tag.pk))
with open(manifest_artifact.file.path, "r") as manifest_file:
manifest_json = json.load(manifest_file)
config_blob = get_or_create_blob(manifest_json["config"], manifest,
path)
manifest.config_blob = config_blob
manifest.save()
new_repo_version.add_content(
Blob.objects.filter(pk=config_blob.pk))
for layer in manifest_json["layers"]:
blob = get_or_create_blob(layer, manifest, path)
new_repo_version.add_content(Blob.objects.filter(pk=blob.pk))
return new_repo_version
def import_collection_from_path(path):
"""
Import a single collection by path.
This method will not fail if the Artifact already exists.
Args:
path: The path to the tarball to import.
"""
artifact = Artifact.init_and_validate(path)
try:
artifact.save()
except IntegrityError:
artifact = Artifact.objects.get(sha256=artifact.sha256)
import_collection(artifact.pk)
async def run(self):
"""
Parse PackageIndex content units.
Ensure, that an uncompressed artifact is available.
"""
with ProgressReport(message="Update PackageIndex units",
code="update.packageindex") as pb:
async for d_content in self.items():
if isinstance(d_content.content, PackageIndex):
if not d_content.d_artifacts:
d_content.content = None
d_content.resolve()
continue
content = d_content.content
if not [
da for da in d_content.d_artifacts
if da.artifact.sha256 == content.sha256
]:
# No main_artifact found, uncompress one
relative_dir = os.path.dirname(
d_content.content.relative_path)
filename = _uncompress_artifact(
d_content.d_artifacts, relative_dir)
da = DeclarativeArtifact(
Artifact.init_and_validate(
filename,
expected_digests={"sha256": content.sha256}),
filename,
content.relative_path,
d_content.d_artifacts[0].remote,
)
d_content.d_artifacts.append(da)
da.artifact.save()
log.info(
"*** Expected: {} *** Uncompressed: {} ***".format(
content.sha256, da.artifact.sha256))
pb.increment()
await self.put(d_content)
def init_content_data(self, serializer, request):
"""Initialize a temporary Artifact."""
shared_resources = []
task_payload = {k: v for k, v in request.data.items()}
file_content = task_payload.pop("file", None)
if file_content:
# in the upload code path make sure, the artifact exists, and the 'file'
# parameter is replaced by an Artifact; this Artifact will be afterwards
# deleted because it serves as a temporary storage for file contents
artifact = Artifact.init_and_validate(file_content)
try:
artifact.save()
except IntegrityError:
# if artifact already exists, let's use it
artifact = Artifact.objects.get(sha256=artifact.sha256)
task_payload["artifact"] = ArtifactSerializer(
artifact, context={"request": request}
).data["pulp_href"]
shared_resources.append(artifact)
return ContentUploadData(shared_resources, task_payload)
def sync(remote_pk, repository_pk):
"""
Sync Collections with ``remote_pk``, and save a new RepositoryVersion for ``repository_pk``.
Args:
remote_pk (str): The remote PK.
repository_pk (str): The repository PK.
Raises:
ValueError: If the remote does not specify a URL to sync or a ``whitelist`` of Collections
to sync.
"""
remote = CollectionRemote.objects.get(pk=remote_pk)
repository = Repository.objects.get(pk=repository_pk)
if not remote.url:
raise ValueError(
_("A CollectionRemote must have a 'url' specified to synchronize.")
)
if not remote.whitelist:
raise ValueError(
_("A CollectionRemote must have a 'whitelist' specified to synchronize."
))
repository_spec_strings = remote.whitelist.split(' ')
def nowhere(*args, **kwargs):
pass
collections_created_pks = []
with tempfile.TemporaryDirectory() as temp_ansible_path:
galaxy_context = GalaxyContext(
collections_path=temp_ansible_path,
server={
'url': remote.url,
'ignore_certs': False,
},
)
install_repository_specs_loop(
display_callback=nowhere,
galaxy_context=galaxy_context,
repository_spec_strings=repository_spec_strings,
)
content_walk_generator = os.walk(temp_ansible_path)
for dirpath, dirnames, filenames in content_walk_generator:
if 'MANIFEST.json' in filenames:
with open(dirpath + os.path.sep +
'MANIFEST.json') as manifest_file:
manifest_data = json.load(manifest_file)
info = manifest_data['collection_info']
filename = '{namespace}-{name}-{version}'.format(
namespace=info['namespace'],
name=info['name'],
version=info['version'],
)
tarfile_path = temp_ansible_path + os.path.sep + filename + '.tar.gz'
with tarfile.open(name=tarfile_path, mode='w|gz') as newtar:
newtar.add(dirpath, arcname=filename)
with transaction.atomic():
collection, created = Collection.objects.get_or_create(
namespace=info['namespace'],
name=info['name'],
version=info['version'])
if created:
artifact = Artifact.init_and_validate(newtar.name)
artifact.save()
ContentArtifact.objects.create(
artifact=artifact,
content=collection,
relative_path=collection.relative_path,
)
collections_created_pks.append(collection)
if collections_created_pks:
with RepositoryVersion.create(repository) as new_version:
collections = Collection.objects.filter(
pk__in=collections_created_pks)
new_version.add_content(collections)
def put(self, request, path, pk=None):
"""
Create a blob from uploaded chunks.
"""
_, repository = self.get_dr_push(request, path)
digest = request.query_params["digest"]
# Try to see if the client came back after we told it to backoff with the ``Throttled``
# exception. In that case we answer based on the task state, or make it backoff again.
# This mechanism seems to work with podman but not with docker. However we let the task run
# anyway, since all clients will look with a HEAD request before attemting to upload a blob
# again.
try:
upload = models.Upload.objects.get(pk=pk, repository=repository)
except models.Upload.DoesNotExist as e_upload:
# Upload has been deleted => task has started or even finished
try:
task = Task.objects.filter(
name__endswith="add_and_remove",
reserved_resources_record__resource=f"upload:{pk}",
).last()
except Task.DoesNotExist:
# No upload and no task for it => the upload probably never existed
# return 404
raise e_upload
if task.state == "completed":
task.delete()
blob = models.Blob.objects.get(digest=digest)
return BlobResponse(blob, path, 201, request)
elif task.state in ["waiting", "running"]:
raise Throttled()
else:
error = task.error
task.delete()
raise Exception(str(error))
chunks = UploadChunk.objects.filter(upload=upload).order_by("offset")
with NamedTemporaryFile("ab") as temp_file:
for chunk in chunks:
temp_file.write(chunk.file.read())
temp_file.flush()
uploaded_file = PulpTemporaryUploadedFile.from_file(File(open(temp_file.name, "rb")))
if uploaded_file.hashers["sha256"].hexdigest() == digest[len("sha256:") :]:
try:
artifact = Artifact.init_and_validate(uploaded_file)
artifact.save()
except IntegrityError:
artifact = Artifact.objects.get(sha256=artifact.sha256)
try:
blob = models.Blob(digest=digest, media_type=models.MEDIA_TYPE.REGULAR_BLOB)
blob.save()
except IntegrityError:
blob = models.Blob.objects.get(digest=digest)
try:
blob_artifact = ContentArtifact(
artifact=artifact, content=blob, relative_path=digest
)
blob_artifact.save()
except IntegrityError:
pass
upload.delete()
dispatched_task = dispatch(
add_and_remove,
[f"upload:{pk}", repository],
kwargs={
"repository_pk": str(repository.pk),
"add_content_units": [str(blob.pk)],
"remove_content_units": [],
},
)
# Wait a small amount of time
for dummy in range(3):
time.sleep(1)
task = Task.objects.get(pk=dispatched_task.pk)
if task.state == "completed":
task.delete()
return BlobResponse(blob, path, 201, request)
elif task.state in ["waiting", "running"]:
continue
else:
error = task.error
task.delete()
raise Exception(str(error))
raise Throttled()
else:
raise Exception("The digest did not match")
async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None,
downloaded=True):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
For non-downloaded content, artifact with its expected checksum and size is created.
"""
if not downloaded:
if not expected_digests:
raise ValueError(
_('No digest is provided for on_demand content creation. Pulp 2 '
'storage path: {}'.format(pulp2_storage_path)))
artifact = Artifact(**expected_digests)
artifact.size = expected_size
return artifact
try:
artifact = Artifact.init_and_validate(
pulp2_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
except (DigestValidationError, FileNotFoundError, SizeValidationError):
if self.skip_corrupted:
_logger.warn(
f'The content located in {pulp2_storage_path} is missing or '
f'corrupted. It was skipped during Pulp 2to3 migration.')
return
raise ArtifactValidationError(
f'The content located in {pulp2_storage_path} is '
f'missing or corrupted. Repair it in pulp2 and re-run '
f'the migration. Alternatively, run migration with '
f'skip_corrupted=True.')
pulp3_storage_relative_path = storage.get_artifact_path(
artifact.sha256)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug(
_('Hard link cannot be created, file will be copied.'))
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
if not expected_digests:
expected_digests = {'sha256': artifact.sha256}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
async def declarative_content_from_git_repo(remote,
url,
git_ref=None,
metadata_only=False):
"""Returns a DeclarativeContent for the Collection in a Git repository."""
if git_ref:
try:
gitrepo = Repo.clone_from(url, uuid4(), depth=1, branch=git_ref)
except GitCommandError:
gitrepo = Repo.clone_from(url, uuid4())
gitrepo.git.checkout(git_ref)
else:
gitrepo = Repo.clone_from(url, uuid4(), depth=1)
commit_sha = gitrepo.head.commit.hexsha
metadata, artifact_path = sync_collection(gitrepo.working_dir, ".")
if not metadata_only:
artifact = Artifact.init_and_validate(artifact_path)
try:
await sync_to_async(artifact.save)()
except IntegrityError:
artifact = Artifact.objects.get(sha256=artifact.sha256)
metadata["artifact_url"] = reverse("artifacts-detail",
args=[artifact.pk])
metadata["artifact"] = artifact
else:
metadata["artifact"] = None
metadata["artifact_url"] = None
metadata["remote_artifact_url"] = "{}/commit/{}".format(
url.rstrip("/"), commit_sha)
artifact = metadata["artifact"]
try:
collection_version = await sync_to_async(
create_collection_from_importer)(metadata,
metadata_only=metadata_only)
await sync_to_async(ContentArtifact.objects.get_or_create)(
artifact=artifact,
content=collection_version,
relative_path=collection_version.relative_path,
)
except ValidationError as e:
if e.args[0]["non_field_errors"][0].code == "unique":
namespace = metadata["metadata"]["namespace"]
name = metadata["metadata"]["name"]
version = metadata["metadata"]["version"]
else:
raise e
collection_version = await sync_to_async(CollectionVersion.objects.get
)(namespace=namespace,
name=name,
version=version)
if artifact is None:
artifact = Artifact()
d_artifact = DeclarativeArtifact(
artifact=artifact,
url=metadata["remote_artifact_url"],
relative_path=collection_version.relative_path,
remote=remote,
deferred_download=metadata_only,
)
# TODO: docs blob support??
d_content = DeclarativeContent(
content=collection_version,
d_artifacts=[d_artifact],
)
return d_content
