def test_stat_no_sha256(self):
"""
Test file stats. No sha256 checksum is available.
Check that stat() works on instance itself
:return:
"""
# Regular File
path = ArtifactoryPath(
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz"
)
constructed_url = (
"http://artifactory.local/artifactory"
"/api/storage"
"/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz")
file_stat = {
"repo": "ext-release-local",
"path": "/org/company/tool/1.0/tool-1.0.tar.gz",
"created": "2014-02-24T21:20:59.999+04:00",
"createdBy": "someuser",
"lastModified": "2014-02-24T21:20:36.000+04:00",
"modifiedBy": "anotheruser",
"lastUpdated": "2014-02-24T21:20:36.000+04:00",
"downloadUri":
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz",
"mimeType": "application/octet-stream",
"size": "26776462",
"checksums": {
"sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095",
"md5": "2af7d54a09e9c36d704cb3a2de28aff3",
},
"originalChecksums": {
"sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095",
"md5": "2af7d54a09e9c36d704cb3a2de28aff3",
},
"uri": constructed_url,
}
responses.add(
responses.GET,
constructed_url,
status=200,
json=file_stat,
)
stats = path.stat()
self.assertEqual(
stats.ctime,
dateutil.parser.parse("2014-02-24T21:20:59.999+04:00"))
self.assertEqual(
stats.mtime,
dateutil.parser.parse("2014-02-24T21:20:36.000+04:00"))
self.assertEqual(stats.created_by, "someuser")
self.assertEqual(stats.modified_by, "anotheruser")
self.assertEqual(stats.mime_type, "application/octet-stream")
self.assertEqual(stats.size, 26776462)
self.assertEqual(stats.sha1,
"fc6c9e8ba6eaca4fa97868ac900570282133c095")
self.assertEqual(stats.sha256, None)
def get_latest_version_age(artifact_url):
path = ArtifactoryPath(artifact_url, apikey=api_key)
properties = ArtifactoryPath.stat(path)
modification_date = properties.mtime
# removing timezone
modification_date = modification_date.replace(tzinfo=None)
aritfact_age = today - modification_date
return aritfact_age.days
def checksum(path, type='md5') -> str:
r"""Calculate checksum for local or remote file.
Args:
path: local file path,
or URL to file path on Artifactory
type: checksum type to calculate,
one of ``'md5'``, ``'sha1'``, ``'sha256'``
Returns:
checksum
Example:
>>> checksum(
... 'https://audeering.jfrog.io/artifactory/'
... 'data-public/emodb/db/1.1.0/db-1.1.0.zip'
... )
'f4cfdbc821a070e1163d225b72b241a7'
"""
if path.startswith('http'):
path = _path(path)
if not path.exists():
raise RuntimeError(f'File not found: {path}')
if type == 'md5':
return ArtifactoryPath.stat(path).md5
elif type == 'sha1':
return ArtifactoryPath.stat(path).sha1
elif type == 'sha256':
return ArtifactoryPath.stat(path).sha256
else:
path = audeer.safe_path(path)
if not os.path.exists(path):
raise RuntimeError(f'File not found: {path}')
if type == 'md5':
return md5sum(path)
elif type == 'sha1':
return sha1sum(path)
elif type == 'sha256':
return sha256sum(path)
def _get_raw_image_digest(self):
manifestpath = '/'.join([
self.artifactory_base,
self._get_artifactory_repo(), # We have to massage the repo for artifactory
self.image.get_image_name(),
self.image.get_tag(),
"manifest.json"
])
manifest_path = ArtifactoryPath(manifestpath, auth=(self.artifactory_user, self.artifactory_key))
try:
return ArtifactoryPath.stat(manifest_path).sha256
except FileNotFoundError as e:
raise ManifestNotFound(e)
def md5(self):
try:
return ArtifactoryPath.stat(self.pkg).md5
except AttributeError:
return md5sum(self.packed_path)
def download(
url: str,
destination: str = '.',
*,
chunk: int = 4 * 1024,
force_download: bool = True,
verbose=False,
) -> str:
r"""Download an artifact.
Args:
url: artifact URL
destination: path to store the artifact,
can be a folder or a file name
chunk: amount of data read at once during the download
force_download: forces the artifact to be downloaded
even if it exists locally already
verbose: show information on the download process
Returns:
path to local artifact
Raises:
RuntimeError: if artifact cannot be found,
or you don't have access rights to the artifact
Example:
>>> file = download(
... (
... 'https://audeering.jfrog.io/artifactory/'
... 'data-public/emodb/db/1.1.0/db-1.1.0.yaml'
... ),
... )
>>> os.path.basename(file)
'db-1.1.0.yaml'
"""
destination = audeer.safe_path(destination)
if os.path.isdir(destination):
destination = os.path.join(destination, os.path.basename(url))
if os.path.exists(destination) and not force_download:
return destination
src_path = _path(url)
if not src_path.exists():
raise RuntimeError(f"Source '{url}' does not exists.")
src_size = ArtifactoryPath.stat(src_path).size
with audeer.progress_bar(total=src_size, disable=not verbose) as pbar:
desc = audeer.format_display_message(
'Download {}'.format(os.path.basename(str(src_path))),
pbar=True,
)
pbar.set_description_str(desc)
pbar.refresh()
try:
dst_size = 0
with src_path.open() as src_fp:
with open(destination, 'wb') as dst_fp:
while src_size > dst_size:
data = src_fp.read(chunk)
n_data = len(data)
if n_data > 0:
dst_fp.write(data)
dst_size += n_data
pbar.update(n_data)
except (KeyboardInterrupt, Exception):
# Clean up broken artifact files
if os.path.exists(destination):
os.remove(destination) # pragma: no cover
raise
return destination