def compile_version_elassandra(version, target_dir, verbose=False, elassandra_version=None): assert_jdk_valid_for_cassandra_version(get_version_from_build(target_dir)) # compiling cassandra and the stress tool logfile = lastlogfilename() logger = get_logger(logfile) common.info("Compiling Elassandra {} ...".format(version)) logger.info("--- Elassandra Build -------------------\n") try: # Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543 # Similar patch seen with buildbot attempt = 0 ret_val = 1 if elassandra_version is None: logger.info("elassandra version is not set, trying with 2.4.2") elassandra_version = "2.4.2" targz_file = "distribution/tar/target/releases/elassandra-%s.tar.gz" % elassandra_version target_install_dir = os.path.join(target_dir, "elassandra-%s") while attempt < 3 and ret_val is not 0: if attempt > 0: logger.info("\n\n`mvn package -DskipTests` failed. Retry #%s...\n\n" % attempt) process = subprocess.Popen([platform_binary('mvn'), 'package', '-DskipTests'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, _, _ = log_info(process, logger) if ret_val is 0: process = subprocess.Popen([platform_binary('tar'), '-xzf', targz_file], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, _, _ = log_info(process, logger) attempt += 1 if ret_val is not 0: raise CCMError('Error compiling Elassandra. See {logfile} or run ' '"ccm showlastlog" for details'.format(logfile=logfile)) except OSError as e: raise CCMError("Error compiling Elassandra. Is maven installed? See %s for details" % logfile)
def compile_version(version, target_dir, verbose=False): assert_jdk_valid_for_cassandra_version(get_version_from_build(target_dir)) # compiling cassandra and the stress tool logfile = lastlogfilename() logger = get_logger(logfile) common.info("Compiling Cassandra {} ...".format(version)) logger.info("--- Cassandra Build -------------------\n") default_build_properties = os.path.join(common.get_default_path(), 'build.properties.default') if os.path.exists(default_build_properties): target_build_properties = os.path.join(target_dir, 'build.properties') logger.info("Copying %s to %s\n" % (default_build_properties, target_build_properties)) shutil.copyfile(default_build_properties, target_build_properties) try: # Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543 # Similar patch seen with buildbot attempt = 0 ret_val = 1 while attempt < 3 and ret_val is not 0: if attempt > 0: logger.info("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt) process = subprocess.Popen([platform_binary('ant'), 'jar'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, _, _ = log_info(process, logger) attempt += 1 if ret_val is not 0: raise CCMError('Error compiling Cassandra. See {logfile} or run ' '"ccm showlastlog" for details'.format(logfile=logfile)) except OSError as e: raise CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile) logger.info("\n\n--- cassandra/stress build ------------\n") stress_dir = os.path.join(target_dir, "tools", "stress") if ( version >= "0.8.0") else \ os.path.join(target_dir, "contrib", "stress") build_xml = os.path.join(stress_dir, 'build.xml') if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1 try: # set permissions correctly, seems to not always be the case stress_bin_dir = os.path.join(stress_dir, 'bin') for f in os.listdir(stress_bin_dir): full_path = os.path.join(stress_bin_dir, f) os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) process = subprocess.Popen([platform_binary('ant'), 'build'], cwd=stress_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, _, _ = log_info(process, logger) if ret_val is not 0: process = subprocess.Popen([platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, _, _ = log_info(process, logger) if ret_val is not 0: raise CCMError("Error compiling Cassandra stress tool. " "See %s for details (you will still be able to use ccm " "but not the stress related commands)" % logfile) except IOError as e: raise CCMError("Error compiling Cassandra stress tool: %s (you will " "still be able to use ccm but not the stress related commands)" % str(e))
def get_relocatable_s3_url(branch, s3_version, links): for reloc_url in links: s3_url = reloc_url.format(branch, s3_version) resp = aws_bucket_ls(s3_url) if resp: return s3_url raise CCMError(f"s3 url was not found for {branch}:{s3_version}")
def download_version(version, url=None, verbose=False, binary=False): """Download, extract, and build Cassandra tarball. if binary == True, download precompiled tarball, otherwise build from source tarball. """ assert_jdk_valid_for_cassandra_version(version) archive_url = ARCHIVE if CCM_CONFIG.has_option('repositories', 'cassandra'): archive_url = CCM_CONFIG.get('repositories', 'cassandra') if binary: archive_url = "%s/%s/apache-cassandra-%s-bin.tar.gz" % ( archive_url, version.split('-')[0], version) if url is None else url else: archive_url = "%s/%s/apache-cassandra-%s-src.tar.gz" % ( archive_url, version.split('-')[0], version) if url is None else url _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(archive_url, target, show_progress=verbose) common.info("Extracting {} as version {} ...".format(target, version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] # pylint: disable=all tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) if binary: # Binary installs don't have a build.xml that is needed # for pulling the version from. Write the version number # into a file to read later in common.get_version_from_build() with open(os.path.join(target_dir, '0.version.txt'), 'w') as f: f.write(version) else: compile_version(version, target_dir, verbose=verbose) except urllib.error.URLError as e: msg = "Invalid version {}".format( version) if url is None else "Invalid url {}".format(url) msg = msg + " (underlying error is: {})".format(str(e)) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: {}".format( str(e))) except CCMError as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: raise CCMError( "Building C* version {} failed. Attempted to delete {} but failed. This will need to be manually deleted" .format(version, target_dir)) raise e
def compile_version(version, target_dir, verbose=False): # compiling cassandra and the stress tool logfile = os.path.join(__get_dir(), "last.log") if verbose: print_("Compiling Cassandra %s ..." % version) with open(logfile, 'w') as lf: lf.write("--- Cassandra Build -------------------\n") try: # Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543 # Similar patch seen with buildbot attempt = 0 ret_val = 1 while attempt < 3 and ret_val is not 0: if attempt > 0: lf.write("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt) ret_val = subprocess.call([platform_binary('ant'),'jar'], cwd=target_dir, stdout=lf, stderr=lf) attempt += 1 if ret_val is not 0: raise CCMError("Error compiling Cassandra. See %s for details" % logfile) except OSError as e: raise CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile) lf.write("\n\n--- cassandra/stress build ------------\n") stress_dir = os.path.join(target_dir, "tools", "stress") if ( version >= "0.8.0") else \ os.path.join(target_dir, "contrib", "stress") build_xml = os.path.join(stress_dir, 'build.xml') if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1 try: # set permissions correctly, seems to not always be the case stress_bin_dir = os.path.join(stress_dir, 'bin') for f in os.listdir(stress_bin_dir): full_path = os.path.join(stress_bin_dir, f) os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) if subprocess.call([platform_binary('ant'), 'build'], cwd=stress_dir, stdout=lf, stderr=lf) is not 0: if subprocess.call([platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=lf, stderr=lf) is not 0: raise CCMError("Error compiling Cassandra stress tool. " "See %s for details (you will still be able to use ccm " "but not the stress related commands)" % logfile) except IOError as e: raise CCMError("Error compiling Cassandra stress tool: %s (you will " "still be able to use ccm but not the stress related commands)" % str(e))
def setup(version, verbose=False): binary = True fallback = True if version.startswith('git:'): clone_development(GIT_REPO, version, verbose=verbose) return (version_directory(version), None) elif version.startswith('local:'): if LOCAL_GIT_REPO is None: raise CCMError("LOCAL_GIT_REPO is not defined!") clone_development(LOCAL_GIT_REPO, version, verbose=verbose) return (version_directory(version), None) elif version.startswith('binary:'): version = version.replace('binary:', '') fallback = False elif version.startswith('github:'): user_name, _ = github_username_and_branch_name(version) clone_development(github_repo_for_user(user_name), version, verbose=verbose) return (directory_name(version), None) elif version.startswith('source:'): version = version.replace('source:', '') binary = False fallback = False if version in ('stable', 'oldstable', 'testing'): version = get_tagged_version_numbers(version)[0] cdir = version_directory(version) if cdir is None: try: download_version(version, verbose=verbose, binary=binary) cdir = version_directory(version) except Exception as e: # If we failed to download from ARCHIVE, # then we build from source from the git repo, # as it is more reliable. # We don't do this if binary: or source: were # explicitly specified. if fallback: common.warning( "Downloading {} failed, due to {}. Trying to build from git instead." .format(version, e)) version = 'git:cassandra-{}'.format(version) clone_development(GIT_REPO, version, verbose=verbose) return (version_directory(version), None) else: raise e return (cdir, version)
def download_version(version, url=None, verbose=False, binary=False): """Download, extract, and build Cassandra tarball. if binary == True, download precompiled tarball, otherwise build from source tarball. """ assert_jdk_valid_for_cassandra_version(version) if binary: u = "%s/%s/apache-cassandra-%s-bin.tar.gz" % ( ARCHIVE, version.split('-')[0], version) if url is None else url else: u = "%s/%s/apache-cassandra-%s-src.tar.gz" % ( ARCHIVE, version.split('-')[0], version) if url is None else url _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(u, target, show_progress=verbose) if verbose: print_("Extracting %s as version %s ..." % (target, version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) if binary: # Binary installs don't have a build.xml that is needed # for pulling the version from. Write the version number # into a file to read later in common.get_version_from_build() with open(os.path.join(target_dir, '0.version.txt'), 'w') as f: f.write(version) else: compile_version(version, target_dir, verbose=verbose) except urllib.error.URLError as e: msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url msg = msg + " (underlying error is: %s)" % str(e) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e)) except CCMError as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) print_("Deleted %s due to error" % target_dir) except: raise CCMError( "Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted" % (version, target_dir)) raise e
def download_version(version, url=None, verbose=False, target_dir=None): """ Download, scylla relocatable package tarballs. """ try: if os.path.exists(url) and url.endswith('.tar.gz'): target = url elif is_valid(url): _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") __download(url, target, show_progress=verbose) else: raise ArgumentError( "unsupported url or file doesn't exist\n\turl={}".format(url)) if verbose: print_("Extracting %s as version %s ..." % (target, version)) tar = tarfile.open(target) tar.extractall(path=target_dir) tar.close() # add breadcrumb so we could list the origin of each part easily for debugging # for example listing all the version we have in ccm scylla-repository # find ~/.ccm/scylla-repository/*/ -iname source.txt | xargs cat source_breadcrumb_file = os.path.join(target_dir, 'source.txt') with open(source_breadcrumb_file, 'w') as f: f.write("version=%s\n" % version) f.write("url=%s\n" % url) except urllib.error.URLError as e: msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url msg = msg + " (underlying error is: %s)" % str(e) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: %s" % str(e)) except CCMError as e: if target_dir: # wipe out the directory if anything goes wrong. try: rmdirs(target_dir) print_("Deleted %s due to error" % target_dir) except: raise CCMError( "Downloading/extracting scylla version %s failed. Attempted to delete %s but failed. This will need to be manually deleted" % (version, target_dir)) raise e
def __download(url, target, username=None, password=None, show_progress=False): if username is not None: password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, url, username, password) handler = urllib.request.HTTPBasicAuthHandler(password_mgr) opener = urllib.request.build_opener(handler) urllib.request.install_opener(opener) u = urllib.request.urlopen(url) f = open(target, 'wb') meta = u.info() file_size = int(meta.get("Content-Length")) if show_progress: print_("Downloading %s to %s (%.3fMB)" % (url, target, float(file_size) / (1024 * 1024))) file_size_dl = 0 block_sz = 8192 status = None attempts = 0 while file_size_dl < file_size: buffer = u.read(block_sz) if not buffer: attempts = attempts + 1 if attempts >= 5: raise CCMError( "Error downloading file (nothing read after %i attempts, downloded only %i of %i bytes)" % (attempts, file_size_dl, file_size)) time.sleep(0.5 * attempts) continue else: attempts = 0 file_size_dl += len(buffer) f.write(buffer) if show_progress: status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size) status = chr(8) * (len(status) + 1) + status print_(status, end='') if show_progress: print_("") f.close() u.close()
def __download(url, target, show_progress=False): u = urllib.request.urlopen(url) f = open(target, 'wb') meta = u.info() file_size = int(meta.getheaders("Content-Length")[0]) if show_progress: print_("Downloading %s to %s (%.3fMB)" % (url, target, float(file_size) / (1024 * 1024))) file_size_dl = 0 block_sz = 8192 status = None attempts = 0 while file_size_dl < file_size: buffer = u.read(block_sz) if not buffer: attempts = attempts + 1 if attempts >= 5: raise CCMError( "Error downloading file (nothing read after %i attempts, downloded only %i of %i bytes)" % (attempts, file_size_dl, file_size)) time.sleep(0.5 * attempts) continue else: attempts = 0 file_size_dl += len(buffer) f.write(buffer) if show_progress: status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size) status = chr(8) * (len(status) + 1) + status print_(status, end='') if show_progress: print_("") f.close() u.close()
def clone_development(git_repo, version, verbose=False): local_git_cache = os.path.join(__get_dir(), '_git_cache') target_dir = os.path.join(__get_dir(), version.replace( ':', '_')) # handle git branches like 'git:trunk'. git_branch = version[4:] # the part of the version after the 'git:' logfile = os.path.join(__get_dir(), "last.log") with open(logfile, 'w') as lf: try: #Checkout/fetch a local repository cache to reduce the number of #remote fetches we need to perform: if not os.path.exists(local_git_cache): if verbose: print_("Cloning Cassandra...") out = subprocess.call( ['git', 'clone', '--mirror', git_repo, local_git_cache], cwd=__get_dir(), stdout=lf, stderr=lf) assert out == 0, "Could not do a git clone" else: if verbose: print_("Fetching Cassandra updates...") out = subprocess.call( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=lf, stderr=lf) #Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. if verbose: print_("Cloning Cassandra (from local cache)") # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) subprocess.call( ['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=lf, stderr=lf) else: subprocess.call( ['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=lf, stderr=lf) # now check out the right version if verbose: print_("Checking out requested branch (%s)" % git_branch) out = subprocess.call(['git', 'checkout', git_branch], cwd=target_dir, stdout=lf, stderr=lf) if int(out) != 0: raise CCMError( "Could not check out git branch %s. Is this a valid branch name? (see last.log for details)" % git_branch) # now compile compile_version(git_branch, target_dir, verbose) else: # branch is already checked out. See if it is behind and recompile if needed. out = subprocess.call(['git', 'fetch', 'origin'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git fetch" status = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=lf).communicate()[0] if str(status).find('[behind') > -1: if verbose: print_("Branch is behind, recompiling") out = subprocess.call(['git', 'pull'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git pull" out = subprocess.call( [platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose) except: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: shutil.rmtree(target_dir) print_("Deleted %s due to error" % target_dir) except: raise CCMError( "Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted" % (version, target_dir)) raise
def setup(version, verbose=False, elassandra_version=None): binary = True fallback = True if version.startswith('git:'): clone_development(GIT_REPO, version, verbose=verbose, elassandra_version=elassandra_version) return (elassandra_version_directory(version, elassandra_version), None) elif version.startswith('local:'): # local: slugs take the form of: "local:/some/path/:somebranch" try: _, path, branch = version.split(':') except ValueError: raise CCMError("local version ({}) appears to be invalid. Please format as local:/some/path/:somebranch".format(version)) clone_development(path, version, verbose=verbose, elassandra_version=elassandra_version) version_dir = elassandra_version_directory(version, elassandra_version) if version_dir is None: raise CCMError("Path provided in local slug appears invalid ({})".format(path)) return (version_dir, None) elif version.startswith('binary:'): version = version.replace('binary:', '') fallback = False elif version.startswith('github:'): user_name, _ = github_username_and_branch_name(version) clone_development(github_repo_for_user(user_name), version, verbose=verbose, elassandra_version=elassandra_version) return (elassandra_version_directory(version, elassandra_version), None) elif version.startswith('source:'): version = version.replace('source:', '') binary = False fallback = False elif version.startswith('alias:'): alias = version.split(":")[1].split("/")[0] try: git_repo = CCM_CONFIG.get("aliases", alias) clone_development(git_repo, version, verbose=verbose, alias=True, elassandra_version=elassandra_version) return (elassandra_version_directory(version, elassandra_version), None) except ConfigParser.NoOptionError as e: common.warning("Unable to find alias {} in configuration file.".format(alias)) raise e if version in ('stable', 'oldstable', 'testing'): version = get_tagged_version_numbers(version)[0] if elassandra_version is None: elassandra_version = version cdir = version_directory(elassandra_version) if cdir is None: try: download_version(elassandra_version, verbose=verbose, binary=binary) cdir = version_directory(elassandra_version) except Exception as e: # If we failed to download from ARCHIVE, # then we build from source from the git repo, # as it is more reliable. # We don't do this if binary: or source: were # explicitly specified. if fallback and False: # skip fallback, would not work with elassandra that way common.warning("Downloading {} failed, due to {}. Trying to build from git instead.".format(version, e)) version = 'git:cassandra-{}'.format(version) clone_development(GIT_REPO, version, verbose=verbose, elassandra_version=elassandra_version) return (version_directory(version), None) else: raise e return (cdir, elassandra_version)
def clone_development(git_repo, version, verbose=False, alias=False, elassandra_version=None): print_(git_repo, version) target_dir = directory_name(version) assert target_dir if 'github' in version: git_repo_name, git_branch = github_username_and_branch_name(version) elif 'local:' in version: git_repo_name = 'local_{}'.format(git_repo) # add git repo location to distinguish cache location for differing repos git_branch = version.split(':')[-1] # last token on 'local:...' slugs should always be branch name elif alias: git_repo_name = 'alias_{}'.format(version.split('/')[0].split(':')[-1]) git_branch = version.split('/')[-1] else: git_repo_name = 'strapdata' git_branch = version.split(':', 1)[1] local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name) logfile = lastlogfilename() logger = get_logger(logfile) try: # Checkout/fetch a local repository cache to reduce the number of # remote fetches we need to perform: if not os.path.exists(local_git_cache): common.info("Cloning Elassandra...") process = subprocess.Popen( ['git', 'clone', '--mirror', git_repo, local_git_cache], cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git clone" else: common.info("Fetching Elassandra updates...") process = subprocess.Popen( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not update git" # Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. common.info("Cloning Elassandra (from local cache)") # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) process = subprocess.Popen( ['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git clone" else: process = subprocess.Popen( ['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git clone" # determine if the request is for a branch is_branch = False try: branch_listing = subprocess.check_output(['git', 'branch', '--all'], cwd=target_dir).decode('utf-8') branches = [b.strip() for b in branch_listing.replace('remotes/origin/', '').split()] is_branch = git_branch in branches except subprocess.CalledProcessError as cpe: common.error("Error Running Branch Filter: {}\nAssumming request is not for a branch".format(cpe.output)) # now check out the right version branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag' common.info("Checking out requested {} ({})".format(branch_or_sha_tag, git_branch)) if is_branch: # we use checkout -B with --track so we can specify that we want to track a specific branch # otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360 # we use -B instead of -b so we reset branches that already exist and create a new one otherwise process = subprocess.Popen(['git', 'checkout', '-B', git_branch, '--track', 'origin/{git_branch}'.format(git_branch=git_branch)], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) else: process = subprocess.Popen( ['git', 'checkout', git_branch], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) if int(out) != 0: raise CCMError('Could not check out git branch {branch}. ' 'Is this a valid branch name? (see {lastlog} or run ' '"ccm showlastlog" for details)'.format( branch=git_branch, lastlog=logfile )) # now compile compile_version(git_branch, target_dir, verbose, elassandra_version=elassandra_version) else: # branch is already checked out. See if it is behind and recompile if needed. process = subprocess.Popen( ['git', 'fetch', 'origin'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git fetch" process = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, status, _ = log_info(process, logger) if str(status).find('[behind') > -1: # If `status` looks like '## cassandra-2.2...origin/cassandra-2.2 [behind 9]\n' common.info("Branch is behind, recompiling") process = subprocess.Popen(['git', 'pull'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git pull" process = subprocess.Popen([platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose, elassandra_version=elassandra_version) elif re.search('\[.*?(ahead|behind).*?\]', status.decode("utf-8")) is not None: # status looks like '## trunk...origin/trunk [ahead 1, behind 29]\n' # If we have diverged in a way that fast-forward merging cannot solve, raise an exception so the cache is wiped common.error("Could not ascertain branch status, please resolve manually.") raise Exception else: # status looks like '## cassandra-2.2...origin/cassandra-2.2\n' common.debug("Branch up to date, not pulling.") except Exception as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: print_('Building C* version {version} failed. Attempted to delete {target_dir}' 'but failed. This will need to be manually deleted'.format( version=version, target_dir=target_dir )) finally: raise e
def compile_version(version, target_dir, verbose=False): # compiling cassandra and the stress tool logfile = lastlogfilename() logger = get_logger(logfile) common.info("Compiling Cassandra {} ...".format(version)) logger.info("--- Cassandra Build -------------------\n") env = update_java_version( install_dir=target_dir, for_build=True, info_message='Cassandra {} build'.format(version)) default_build_properties = os.path.join(common.get_default_path(), 'build.properties.default') if os.path.exists(default_build_properties): target_build_properties = os.path.join(target_dir, 'build.properties') logger.info("Copying %s to %s\n" % (default_build_properties, target_build_properties)) shutil.copyfile(default_build_properties, target_build_properties) try: # Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543 # Similar patch seen with buildbot attempt = 0 ret_val = 1 gradlew = os.path.join(target_dir, platform_binary('gradlew')) if os.path.exists(gradlew): cmd = [gradlew, 'jar'] else: # No gradle, use ant cmd = [platform_binary('ant'), 'jar'] if get_jdk_version_int() >= 11: cmd.append('-Duse.jdk11=true') while attempt < 3 and ret_val != 0: if attempt > 0: logger.info("\n\n`{}` failed. Retry #{}...\n\n".format( ' '.join(cmd), attempt)) process = subprocess.Popen(cmd, cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, stdout, stderr = log_info(process, logger) attempt += 1 if ret_val != 0: raise CCMError( 'Error compiling Cassandra. See {logfile} or run ' '"ccm showlastlog" for details, stdout=\'{stdout}\' stderr=\'{stderr}\'' .format(logfile=logfile, stdout=stdout.decode(), stderr=stderr.decode())) except OSError as e: raise CCMError( "Error compiling Cassandra. Is ant installed? See %s for details" % logfile) stress_dir = os.path.join(target_dir, "tools", "stress") if ( version >= "0.8.0") else \ os.path.join(target_dir, "contrib", "stress") build_xml = os.path.join(stress_dir, 'build.xml') if os.path.exists( build_xml): # building stress separately is only necessary pre-1.1 logger.info("\n\n--- cassandra/stress build ------------\n") try: # set permissions correctly, seems to not always be the case stress_bin_dir = os.path.join(stress_dir, 'bin') for f in os.listdir(stress_bin_dir): full_path = os.path.join(stress_bin_dir, f) os.chmod( full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) process = subprocess.Popen([platform_binary('ant'), 'build'], cwd=stress_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, _, _ = log_info(process, logger) if ret_val != 0: process = subprocess.Popen( [platform_binary('ant'), 'stress-build'], cwd=target_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret_val, _, _ = log_info(process, logger) if ret_val != 0: raise CCMError( "Error compiling Cassandra stress tool. " "See %s for details (you will still be able to use ccm " "but not the stress related commands)" % logfile) except IOError as e: raise CCMError( "Error compiling Cassandra stress tool: %s (you will " "still be able to use ccm but not the stress related commands)" % str(e))
def setup(version, verbose=False): binary = True fallback = True if version.startswith('git:'): clone_development(GIT_REPO, version, verbose=verbose) return (version_directory(version), None) elif version.startswith('local:'): # local: slugs take the form of: "local:/some/path/:somebranch" try: _, path, branch = version.split(':') except ValueError: raise CCMError( "local version ({}) appears to be invalid. Please format as local:/some/path/:somebranch" .format(version)) clone_development(path, version, verbose=verbose) version_dir = version_directory(version) if version_dir is None: raise CCMError( "Path provided in local slug appears invalid ({})".format( path)) return (version_dir, None) elif version.startswith('binary:'): version = version.replace('binary:', '') fallback = False elif version.startswith('github:'): user_name, _ = github_username_and_branch_name(version) # make sure to use http for cloning read-only repos such as 'github:apache/cassandra-2.1' if user_name == "apache": clone_development(GITHUB_REPO, version, verbose=verbose) else: clone_development(github_repo_for_user(user_name), version, verbose=verbose) return (directory_name(version), None) elif version.startswith('source:'): version = version.replace('source:', '') elif version.startswith('clone:'): # locally present C* source tree version = version.replace('clone:', '') return (version, None) elif version.startswith('alias:'): alias = version.split(":")[1].split("/")[0] try: git_repo = CCM_CONFIG.get("aliases", alias) clone_development(git_repo, version, verbose=verbose, alias=True) return (directory_name(version), None) except ConfigParser.NoOptionError as e: common.warning( "Unable to find alias {} in configuration file.".format(alias)) raise e if version in ('stable', 'oldstable', 'testing'): version = get_tagged_version_numbers(version)[0] cdir = version_directory(version) if cdir is None: try: download_version(version, verbose=verbose, binary=binary) cdir = version_directory(version) except Exception as e: # If we failed to download from ARCHIVE, # then we build from source from the git repo, # as it is more reliable. # We don't do this if binary: or source: were # explicitly specified. if fallback: common.warning( "Downloading {} failed, trying to build from git instead.\n" "The error was: {}".format(version, e)) version = 'git:cassandra-{}'.format(version) clone_development(GIT_REPO, version, verbose=verbose) return (version_directory(version), None) else: raise e return (cdir, version)
def clone_development(git_repo, version, verbose=False): print_(git_repo, version) target_dir = directory_name(version) assert target_dir if 'github' in version: git_repo_name, git_branch = github_username_and_branch_name(version) else: git_repo_name = 'apache' git_branch = version.split(':', 1)[1] local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name) logfile = lastlogfilename() with open(logfile, 'w') as lf: try: # Checkout/fetch a local repository cache to reduce the number of # remote fetches we need to perform: if not os.path.exists(local_git_cache): if verbose: print_("Cloning Cassandra...") out = subprocess.call( ['git', 'clone', '--mirror', git_repo, local_git_cache], cwd=__get_dir(), stdout=lf, stderr=lf) assert out == 0, "Could not do a git clone" else: if verbose: print_("Fetching Cassandra updates...") out = subprocess.call( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=lf, stderr=lf) # Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. if verbose: print_("Cloning Cassandra (from local cache)") # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) subprocess.call( ['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=lf, stderr=lf) else: subprocess.call( ['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=lf, stderr=lf) # determine if the request is for a branch is_branch = False try: branch_listing = subprocess.check_output( ['git', 'branch', '--all'], cwd=target_dir).decode('utf-8') branches = [ b.strip() for b in branch_listing.replace( 'remotes/origin/', '').split() ] is_branch = git_branch in branches except subprocess.CalledProcessError as cpe: print_( "Error Running Branch Filter: {}\nAssumming request is not for a branch" .format(cpe.output)) # now check out the right version if verbose: branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag' print_("Checking out requested {} ({})".format( branch_or_sha_tag, git_branch)) if is_branch: # we use checkout -B with --track so we can specify that we want to track a specific branch # otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360 # we use -B instead of -b so we reset branches that already exist and create a new one otherwise out = subprocess.call([ 'git', 'checkout', '-B', git_branch, '--track', 'origin/{git_branch}'.format(git_branch=git_branch) ], cwd=target_dir, stdout=lf, stderr=lf) else: out = subprocess.call(['git', 'checkout', git_branch], cwd=target_dir, stdout=lf, stderr=lf) if int(out) != 0: raise CCMError( 'Could not check out git branch {branch}. ' 'Is this a valid branch name? (see {lastlog} or run ' '"ccm showlastlog" for details)'.format( branch=git_branch, lastlog=logfile)) # now compile compile_version(git_branch, target_dir, verbose) else: # branch is already checked out. See if it is behind and recompile if needed. out = subprocess.call(['git', 'fetch', 'origin'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git fetch" status = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=lf).communicate()[0] if str(status).find('[behind') > -1: if verbose: print_("Branch is behind, recompiling") out = subprocess.call(['git', 'pull'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git pull" out = subprocess.call( [platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose) except: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) print_("Deleted %s due to error" % target_dir) except: raise CCMError( "Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted" % (version, target_dir)) raise