def download_dse_version(version, username, password, verbose=False): url = DSE_ARCHIVE % version _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(url, target, username=username, password=password, show_progress=verbose) if verbose: print_("Extracting %s as version %s ..." % (target, version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): shutil.rmtree(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) except urllib.error.URLError as e: msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url msg = msg + " (underlying error is: %s)" % str(e) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e))
def download_version(version, url=None, verbose=False, target_dir=None): """ Download, scylla relocatable package tarballs. """ try: if os.path.exists(url) and url.endswith('.tar.gz'): target = url elif is_valid(url): _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") __download(url, target, show_progress=verbose) else: raise ArgumentError( "unsupported url or file doesn't exist\n\turl={}".format(url)) if verbose: print_("Extracting %s as version %s ..." % (target, version)) tar = tarfile.open(target) tar.extractall(path=target_dir) tar.close() # add breadcrumb so we could list the origin of each part easily for debugging # for example listing all the version we have in ccm scylla-repository # find ~/.ccm/scylla-repository/*/ -iname source.txt | xargs cat source_breadcrumb_file = os.path.join(target_dir, 'source.txt') with open(source_breadcrumb_file, 'w') as f: f.write("version=%s\n" % version) f.write("url=%s\n" % url) except urllib.error.URLError as e: msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url msg = msg + " (underlying error is: %s)" % str(e) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError( "Unable to uncompress downloaded file: %s" % str(e))
def download_opscenter_version(version, username, password, target_version, verbose=False): url = OPSC_ARCHIVE if CCM_CONFIG.has_option('repositories', 'opscenter'): url = CCM_CONFIG.get('repositories', 'opscenter') url = url % version _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: if username is None: common.warning("No dse username detected, specify one using --dse-username or passing in a credentials file using --dse-credentials.") if password is None: common.warning("No dse password detected, specify one using --dse-password or passing in a credentials file using --dse-credentials.") __download(url, target, username=username, password=password, show_progress=verbose) common.info("Extracting {} as version {} ...".format(target, target_version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] # pylint: disable=all tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), target_version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) except urllib.error.URLError as e: msg = "Invalid version {}".format(version) if url is None else "Invalid url {}".format(url) msg = msg + " (underlying error is: {})".format(str(e)) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: {}".format(str(e)))
def download_dse_version(version, username, password, verbose=False): url = DSE_ARCHIVE % version _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: if username is None: common.warning( "No dse username detected, specify one using --dse-username or passing in a credentials file using --dse-credentials." ) if password is None: common.warning( "No dse password detected, specify one using --dse-password or passing in a credentials file using --dse-credentials." ) __download(url, target, username=username, password=password, show_progress=verbose) common.debug("Extracting {} as version {} ...".format(target, version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) except urllib.error.URLError as e: msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url msg = msg + " (underlying error is: %s)" % str(e) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e))
def download_version(version, url=None, verbose=False, binary=False): """Download, extract, and build Cassandra tarball. if binary == True, download precompiled tarball, otherwise build from source tarball. """ assert_jdk_valid_for_cassandra_version(version) archive_url = ARCHIVE if CCM_CONFIG.has_option('repositories', 'cassandra'): archive_url = CCM_CONFIG.get('repositories', 'cassandra') if binary: archive_url = "%s/%s/apache-cassandra-%s-bin.tar.gz" % ( archive_url, version.split('-')[0], version) if url is None else url else: archive_url = "%s/%s/apache-cassandra-%s-src.tar.gz" % ( archive_url, version.split('-')[0], version) if url is None else url _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(archive_url, target, show_progress=verbose) common.info("Extracting {} as version {} ...".format(target, version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] # pylint: disable=all tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) if binary: # Binary installs don't have a build.xml that is needed # for pulling the version from. Write the version number # into a file to read later in common.get_version_from_build() with open(os.path.join(target_dir, '0.version.txt'), 'w') as f: f.write(version) else: compile_version(version, target_dir, verbose=verbose) except urllib.error.URLError as e: msg = "Invalid version {}".format( version) if url is None else "Invalid url {}".format(url) msg = msg + " (underlying error is: {})".format(str(e)) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: {}".format( str(e))) except CCMError as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: raise CCMError( "Building C* version {} failed. Attempted to delete {} but failed. This will need to be manually deleted" .format(version, target_dir)) raise e
def download_version(version, url=None, verbose=False, target_dir=None): """ Download, scylla relocatable package tarballs. """ try: if os.path.exists(url) and url.endswith('.tar.gz'): target = url elif is_valid(url): _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") __download(url, target, show_progress=verbose) else: raise ArgumentError( "unsupported url or file doesn't exist\n\turl={}".format(url)) if verbose: print_("Extracting %s as version %s ..." % (target, version)) tar = tarfile.open(target) tar.extractall(path=target_dir) tar.close() # if relocatable package format >= 2, need to extract files under subdir package_version_file = "{}/.relocatable_package_version".format( target_dir) if os.path.exists(package_version_file): with open(package_version_file) as f: package_version = packaging.version.parse(f.read().strip()) if package_version > packaging.version.parse('2.1'): print( f'Unknown relocatable package format version: {package_version}' ) sys.exit(1) print( f'Relocatable package format version {package_version} detected.' ) pkg_dir = glob.glob('{}/*/'.format(target_dir))[0] shutil.move(str(pkg_dir), target_dir + '.new') shutil.rmtree(target_dir) shutil.move(target_dir + '.new', target_dir) else: package_version = packaging.version.parse('1') print('Legacy relocatable package format detected.') # add breadcrumb so we could list the origin of each part easily for debugging # for example listing all the version we have in ccm scylla-repository # find ~/.ccm/scylla-repository/*/ -iname source.txt | xargs cat source_breadcrumb_file = os.path.join(target_dir, 'source.txt') with open(source_breadcrumb_file, 'w') as f: f.write("version=%s\n" % version) f.write("url=%s\n" % url) return package_version except urllib.error.URLError as e: msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url msg = msg + " (underlying error is: %s)" % str(e) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e))
def download_version(version, url=None, verbose=False, binary=False): """Download, extract, and build Cassandra tarball. if binary == True, download precompiled tarball, otherwise build from source tarball. """ assert_jdk_valid_for_cassandra_version(version) if binary: u = "%s/%s/apache-cassandra-%s-bin.tar.gz" % ( ARCHIVE, version.split('-')[0], version) if url is None else url else: u = "%s/%s/apache-cassandra-%s-src.tar.gz" % ( ARCHIVE, version.split('-')[0], version) if url is None else url _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(u, target, show_progress=verbose) if verbose: print_("Extracting %s as version %s ..." % (target, version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) if binary: # Binary installs don't have a build.xml that is needed # for pulling the version from. Write the version number # into a file to read later in common.get_version_from_build() with open(os.path.join(target_dir, '0.version.txt'), 'w') as f: f.write(version) else: compile_version(version, target_dir, verbose=verbose) except urllib.error.URLError as e: msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url msg = msg + " (underlying error is: %s)" % str(e) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e)) except CCMError as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) print_("Deleted %s due to error" % target_dir) except: raise CCMError( "Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted" % (version, target_dir)) raise e
def download_opscenter_version(version, target_version, verbose=False): url = OPSC_ARCHIVE % version _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(url, target, show_progress=verbose) common.info("Extracting {} as version {} ...".format(target, target_version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] # pylint: disable=all tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), target_version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) except urllib.error.URLError as e: msg = "Invalid version {}".format(version) if url is None else "Invalid url {}".format(url) msg = msg + " (underlying error is: {})".format(str(e)) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: {}".format(str(e)))