async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None,
downloaded=True):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
For non-downloaded content, artifact with its expected checksum and size is created.
"""
if not downloaded:
if not expected_digests:
raise ValueError(
_('No digest is provided for on_demand content creation. Pulp 2 '
'storage path: {}'.format(pulp2_storage_path)))
artifact = Artifact(**expected_digests)
artifact.size = expected_size
return artifact
artifact = Artifact.init_and_validate(
pulp2_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
pulp3_storage_relative_path = storage.get_artifact_path(
artifact.sha256)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug(
_('Hard link cannot be created, file will be copied.'))
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
if not expected_digests:
expected_digests = {'sha256': artifact.sha256}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
def _artifact_from_data(raw_data):
tmpfile = PulpTemporaryUploadedFile("", "application/octet-stream",
len(raw_data), "", "")
tmpfile.write(raw_data)
for hasher in Artifact.DIGEST_FIELDS:
tmpfile.hashers[hasher].update(raw_data)
artifact = Artifact()
artifact.file = tmpfile
artifact.size = tmpfile.size
for hasher in Artifact.DIGEST_FIELDS:
setattr(artifact, hasher, tmpfile.hashers[hasher].hexdigest())
artifact.save()
return artifact
async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None,
downloaded=True):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
For non-downloaded content, artifact with its expected checksum and size is created.
"""
if not downloaded:
if not expected_digests:
raise ValueError(
_('No digest is provided for on_demand content creation. Pulp 2 '
'storage path: {}'.format(pulp2_storage_path)))
artifact = Artifact(**expected_digests)
artifact.size = expected_size
return artifact
try:
artifact = Artifact.init_and_validate(
pulp2_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
except (DigestValidationError, FileNotFoundError, SizeValidationError):
if self.skip_corrupted:
_logger.warn(
f'The content located in {pulp2_storage_path} is missing or '
f'corrupted. It was skipped during Pulp 2to3 migration.')
return
raise ArtifactValidationError(
f'The content located in {pulp2_storage_path} is '
f'missing or corrupted. Repair it in pulp2 and re-run '
f'the migration. Alternatively, run migration with '
f'skip_corrupted=True.')
pulp3_storage_relative_path = storage.get_artifact_path(
artifact.sha256)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug(
_('Hard link cannot be created, file will be copied.'))
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
if not expected_digests:
expected_digests = {'sha256': artifact.sha256}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
