def download_version(version, url=None, verbose=False, target_dir=None):
"""
Download, scylla relocatable package tarballs.
"""
try:
if os.path.exists(url) and url.endswith('.tar.gz'):
target = url
elif is_valid(url):
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
res = download_version_from_s3(url=url, target_path=target,verbose=verbose)
if not res:
download_file(url=url, target_path=target,verbose=verbose)
else:
raise ArgumentError(
"unsupported url or file doesn't exist\n\turl={}".format(url))
if verbose:
print_(f"Extracting {target} ({url}, {target_dir}) as version {version} ...")
tar = tarfile.open(target)
tar.extractall(path=target_dir)
tar.close()
# if relocatable package format >= 2, need to extract files under subdir
package_version_file = "{}/.relocatable_package_version".format(target_dir)
if os.path.exists(package_version_file):
with open(package_version_file) as f:
package_version = packaging.version.parse(f.read().strip())
if package_version > packaging.version.parse('2.1'):
print(f'Unknown relocatable package format version: {package_version}')
sys.exit(1)
print(f'Relocatable package format version {package_version} detected.')
pkg_dir = glob.glob('{}/*/'.format(target_dir))[0]
shutil.move(str(pkg_dir), target_dir + '.new')
shutil.rmtree(target_dir)
shutil.move(target_dir + '.new', target_dir)
else:
package_version = packaging.version.parse('1')
print('Legacy relocatable package format detected.')
# add breadcrumb so we could list the origin of each part easily for debugging
# for example listing all the version we have in ccm scylla-repository
# find ~/.ccm/scylla-repository/*/ -iname source.txt | xargs cat
source_breadcrumb_file = os.path.join(target_dir, 'source.txt')
with open(source_breadcrumb_file, 'w') as f:
f.write("version=%s\n" % version)
f.write("url=%s\n" % url)
return package_version
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError(
"Unable to uncompress downloaded file: %s" % str(e))
def test_download_version_from_s3_non_exist_file(self, tmpdir):
res = download_version_from_s3("https://s3.amazonaws.com/downloads.scylladb.com/abcdefg",
target_path=pathlib.Path(tmpdir) / 'scylla-manager.repo')
assert res is None
def test_download_version_from_s3(self, tmpdir):
target_path = str(pathlib.Path(tmpdir) / 'scylla-manager.repo')
res = download_version_from_s3("https://s3.amazonaws.com/downloads.scylladb.com/manager/rpm/unstable/centos/master/latest/scylla-manager.repo",
target_path=target_path)
assert res == target_path
@pytest.mark.repo_tests
class TestUtilsDownload:
def test_download_version_from_s3(self, tmpdir):
target_path = str(pathlib.Path(tmpdir) / 'scylla-manager.repo')
res = download_version_from_s3("https://s3.amazonaws.com/downloads.scylladb.com/manager/rpm/unstable/centos/master/latest/scylla-manager.repo",
target_path=target_path)
assert res == target_path
def test_download_version_from_s3_non_exist_file(self, tmpdir):
res = download_version_from_s3("https://s3.amazonaws.com/downloads.scylladb.com/abcdefg",
target_path=pathlib.Path(tmpdir) / 'scylla-manager.repo')
assert res is None
def test_download_file(self, tmpdir):
target_path = str(pathlib.Path(tmpdir) / 'scylla-manager.repo')
res = download_file("https://s3.amazonaws.com/downloads.scylladb.com/manager/rpm/unstable/centos/master/latest/scylla-manager.repo",
target_path=target_path)
assert res == target_path
def test_download_file_non_exist_file(self, tmpdir):
with pytest.raises(requests.exceptions.HTTPError, match='Not Found'):
download_file("https://s3.amazonaws.com/downloads.scylladb.com/abcdefg",
target_path=pathlib.Path(tmpdir) / 'scylla-manager.repo')
if __name__ == '__main__':
# demo of progress bar
download_version_from_s3(
'https://s3.amazonaws.com/downloads.scylladb.com/relocatable/unstable/master/2020-08-29T22:24:05Z/scylla-package.tar.gz',
target_path='tempfile', verbose=True)