def storage_path(self, name):
"""
Callable used by FileField to determine where the uploaded file should be stored.
Args:
name (str): Original name of uploaded file. It is ignored by this method because the
sha256 checksum is used to determine a file path instead.
"""
return storage.get_artifact_path(self.sha256)
def storage_path(self, name):
"""
Callable used by FileField to determine where the uploaded file should be stored.
Args:
name (str): Original name of uploaded file. It is ignored by this method because the
sha256 checksum is used to determine a file path instead.
"""
return storage.get_artifact_path(self.sha256)
async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None,
downloaded=True):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
For non-downloaded content, artifact with its expected checksum and size is created.
"""
if not downloaded:
if not expected_digests:
raise ValueError(
_('No digest is provided for on_demand content creation. Pulp 2 '
'storage path: {}'.format(pulp2_storage_path)))
artifact = Artifact(**expected_digests)
artifact.size = expected_size
return artifact
artifact = Artifact.init_and_validate(
pulp2_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
pulp3_storage_relative_path = storage.get_artifact_path(
artifact.sha256)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug(
_('Hard link cannot be created, file will be copied.'))
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
if not expected_digests:
expected_digests = {'sha256': artifact.sha256}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
"""
if not expected_digests.get('sha256'):
# TODO: all checksums are calculated for the pulp 2 storage path, is it ok?
artifact = Artifact.init_and_validate(pulp2_storage_path,
size=expected_size)
sha256digest = expected_digests.get('sha256') or artifact.sha256
pulp3_storage_relative_path = storage.get_artifact_path(sha256digest)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug('Hard link cannot be created, file will be copied.')
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
expected_digests = {'sha256': sha256digest}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
async def create_artifact(self,
pulp2_storage_path,
expected_digests={},
expected_size=None,
downloaded=True):
"""
Create a hard link if possible and then create an Artifact.
If it's not possible to create a hard link, file is copied to the Pulp 3 storage.
For non-downloaded content, artifact with its expected checksum and size is created.
"""
if not downloaded:
if not expected_digests:
raise ValueError(
_('No digest is provided for on_demand content creation. Pulp 2 '
'storage path: {}'.format(pulp2_storage_path)))
artifact = Artifact(**expected_digests)
artifact.size = expected_size
return artifact
try:
artifact = Artifact.init_and_validate(
pulp2_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
except (DigestValidationError, FileNotFoundError, SizeValidationError):
if self.skip_corrupted:
_logger.warn(
f'The content located in {pulp2_storage_path} is missing or '
f'corrupted. It was skipped during Pulp 2to3 migration.')
return
raise ArtifactValidationError(
f'The content located in {pulp2_storage_path} is '
f'missing or corrupted. Repair it in pulp2 and re-run '
f'the migration. Alternatively, run migration with '
f'skip_corrupted=True.')
pulp3_storage_relative_path = storage.get_artifact_path(
artifact.sha256)
pulp3_storage_path = os.path.join(settings.MEDIA_ROOT,
pulp3_storage_relative_path)
os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True)
is_copied = False
try:
os.link(pulp2_storage_path, pulp3_storage_path)
except FileExistsError:
pass
except OSError:
_logger.debug(
_('Hard link cannot be created, file will be copied.'))
shutil.copy2(pulp2_storage_path, pulp3_storage_path)
is_copied = True
if not expected_digests:
expected_digests = {'sha256': artifact.sha256}
if is_copied:
# recalculate checksums to ensure that after being copied a file is still fine
artifact = Artifact.init_and_validate(
file=pulp3_storage_path,
expected_digests=expected_digests,
expected_size=expected_size)
else:
# a hard link has been created or a file has already been in the pulp 3 storage, so
# artifact's path can be just updated and no checksum recalculation is needed.
artifact.file = pulp3_storage_path
return artifact
