def download_dse_version(version, username, password, verbose=False):
url = DSE_ARCHIVE % version
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
__download(url,
target,
username=username,
password=password,
show_progress=verbose)
if verbose:
print_("Extracting %s as version %s ..." % (target, version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0]
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), version)
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: %s" %
str(e))
def download_version(version, url=None, verbose=False, target_dir=None):
"""
Download, scylla relocatable package tarballs.
"""
try:
if os.path.exists(url) and url.endswith('.tar.gz'):
target = url
elif is_valid(url):
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
__download(url, target, show_progress=verbose)
else:
raise ArgumentError(
"unsupported url or file doesn't exist\n\turl={}".format(url))
if verbose:
print_("Extracting %s as version %s ..." % (target, version))
tar = tarfile.open(target)
tar.extractall(path=target_dir)
tar.close()
# add breadcrumb so we could list the origin of each part easily for debugging
# for example listing all the version we have in ccm scylla-repository
# find ~/.ccm/scylla-repository/*/ -iname source.txt | xargs cat
source_breadcrumb_file = os.path.join(target_dir, 'source.txt')
with open(source_breadcrumb_file, 'w') as f:
f.write("version=%s\n" % version)
f.write("url=%s\n" % url)
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError(
"Unable to uncompress downloaded file: %s" % str(e))
def download_opscenter_version(version, username, password, target_version, verbose=False):
url = OPSC_ARCHIVE
if CCM_CONFIG.has_option('repositories', 'opscenter'):
url = CCM_CONFIG.get('repositories', 'opscenter')
url = url % version
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
if username is None:
common.warning("No dse username detected, specify one using --dse-username or passing in a credentials file using --dse-credentials.")
if password is None:
common.warning("No dse password detected, specify one using --dse-password or passing in a credentials file using --dse-credentials.")
__download(url, target, username=username, password=password, show_progress=verbose)
common.info("Extracting {} as version {} ...".format(target, target_version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0] # pylint: disable=all
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), target_version)
if os.path.exists(target_dir):
rmdirs(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
except urllib.error.URLError as e:
msg = "Invalid version {}".format(version) if url is None else "Invalid url {}".format(url)
msg = msg + " (underlying error is: {})".format(str(e))
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: {}".format(str(e)))
def download_dse_version(version, username, password, verbose=False):
url = DSE_ARCHIVE % version
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
if username is None:
common.warning(
"No dse username detected, specify one using --dse-username or passing in a credentials file using --dse-credentials."
)
if password is None:
common.warning(
"No dse password detected, specify one using --dse-password or passing in a credentials file using --dse-credentials."
)
__download(url,
target,
username=username,
password=password,
show_progress=verbose)
common.debug("Extracting {} as version {} ...".format(target, version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0]
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), version)
if os.path.exists(target_dir):
rmdirs(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: %s" %
str(e))
def download_version(version, url=None, verbose=False, binary=False):
"""Download, extract, and build Cassandra tarball.
if binary == True, download precompiled tarball, otherwise build from source tarball.
"""
assert_jdk_valid_for_cassandra_version(version)
archive_url = ARCHIVE
if CCM_CONFIG.has_option('repositories', 'cassandra'):
archive_url = CCM_CONFIG.get('repositories', 'cassandra')
if binary:
archive_url = "%s/%s/apache-cassandra-%s-bin.tar.gz" % (
archive_url, version.split('-')[0],
version) if url is None else url
else:
archive_url = "%s/%s/apache-cassandra-%s-src.tar.gz" % (
archive_url, version.split('-')[0],
version) if url is None else url
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
__download(archive_url, target, show_progress=verbose)
common.info("Extracting {} as version {} ...".format(target, version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0] # pylint: disable=all
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), version)
if os.path.exists(target_dir):
rmdirs(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
if binary:
# Binary installs don't have a build.xml that is needed
# for pulling the version from. Write the version number
# into a file to read later in common.get_version_from_build()
with open(os.path.join(target_dir, '0.version.txt'), 'w') as f:
f.write(version)
else:
compile_version(version, target_dir, verbose=verbose)
except urllib.error.URLError as e:
msg = "Invalid version {}".format(
version) if url is None else "Invalid url {}".format(url)
msg = msg + " (underlying error is: {})".format(str(e))
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: {}".format(
str(e)))
except CCMError as e:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
rmdirs(target_dir)
common.error("Deleted {} due to error".format(target_dir))
except:
raise CCMError(
"Building C* version {} failed. Attempted to delete {} but failed. This will need to be manually deleted"
.format(version, target_dir))
raise e
def download_version(version, url=None, verbose=False, target_dir=None):
"""
Download, scylla relocatable package tarballs.
"""
try:
if os.path.exists(url) and url.endswith('.tar.gz'):
target = url
elif is_valid(url):
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
__download(url, target, show_progress=verbose)
else:
raise ArgumentError(
"unsupported url or file doesn't exist\n\turl={}".format(url))
if verbose:
print_("Extracting %s as version %s ..." % (target, version))
tar = tarfile.open(target)
tar.extractall(path=target_dir)
tar.close()
# if relocatable package format >= 2, need to extract files under subdir
package_version_file = "{}/.relocatable_package_version".format(
target_dir)
if os.path.exists(package_version_file):
with open(package_version_file) as f:
package_version = packaging.version.parse(f.read().strip())
if package_version > packaging.version.parse('2.1'):
print(
f'Unknown relocatable package format version: {package_version}'
)
sys.exit(1)
print(
f'Relocatable package format version {package_version} detected.'
)
pkg_dir = glob.glob('{}/*/'.format(target_dir))[0]
shutil.move(str(pkg_dir), target_dir + '.new')
shutil.rmtree(target_dir)
shutil.move(target_dir + '.new', target_dir)
else:
package_version = packaging.version.parse('1')
print('Legacy relocatable package format detected.')
# add breadcrumb so we could list the origin of each part easily for debugging
# for example listing all the version we have in ccm scylla-repository
# find ~/.ccm/scylla-repository/*/ -iname source.txt | xargs cat
source_breadcrumb_file = os.path.join(target_dir, 'source.txt')
with open(source_breadcrumb_file, 'w') as f:
f.write("version=%s\n" % version)
f.write("url=%s\n" % url)
return package_version
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: %s" %
str(e))
def download_version(version, url=None, verbose=False, binary=False):
"""Download, extract, and build Cassandra tarball.
if binary == True, download precompiled tarball, otherwise build from source tarball.
"""
assert_jdk_valid_for_cassandra_version(version)
if binary:
u = "%s/%s/apache-cassandra-%s-bin.tar.gz" % (
ARCHIVE, version.split('-')[0], version) if url is None else url
else:
u = "%s/%s/apache-cassandra-%s-src.tar.gz" % (
ARCHIVE, version.split('-')[0], version) if url is None else url
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
__download(u, target, show_progress=verbose)
if verbose:
print_("Extracting %s as version %s ..." % (target, version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0]
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), version)
if os.path.exists(target_dir):
rmdirs(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
if binary:
# Binary installs don't have a build.xml that is needed
# for pulling the version from. Write the version number
# into a file to read later in common.get_version_from_build()
with open(os.path.join(target_dir, '0.version.txt'), 'w') as f:
f.write(version)
else:
compile_version(version, target_dir, verbose=verbose)
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: %s" %
str(e))
except CCMError as e:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
rmdirs(target_dir)
print_("Deleted %s due to error" % target_dir)
except:
raise CCMError(
"Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted"
% (version, target_dir))
raise e
def download_opscenter_version(version, target_version, verbose=False):
url = OPSC_ARCHIVE % version
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
__download(url, target, show_progress=verbose)
common.info("Extracting {} as version {} ...".format(target, target_version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0] # pylint: disable=all
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), target_version)
if os.path.exists(target_dir):
rmdirs(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
except urllib.error.URLError as e:
msg = "Invalid version {}".format(version) if url is None else "Invalid url {}".format(url)
msg = msg + " (underlying error is: {})".format(str(e))
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: {}".format(str(e)))