def download_version(version, url=None, verbose=False, binary=False): """Download, extract, and build Cassandra tarball. if binary == True, download precompiled tarball, otherwise build from source tarball. """ assert_jdk_valid_for_cassandra_version(version) archive_url = ARCHIVE if CCM_CONFIG.has_option('repositories', 'cassandra'): archive_url = CCM_CONFIG.get('repositories', 'cassandra') if binary: archive_url = "%s/%s/apache-cassandra-%s-bin.tar.gz" % ( archive_url, version.split('-')[0], version) if url is None else url else: archive_url = "%s/%s/apache-cassandra-%s-src.tar.gz" % ( archive_url, version.split('-')[0], version) if url is None else url _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(archive_url, target, show_progress=verbose) common.info("Extracting {} as version {} ...".format(target, version)) tar = tarfile.open(target) dir = tar.next().name.split("/")[0] # pylint: disable=all tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) if binary: # Binary installs don't have a build.xml that is needed # for pulling the version from. Write the version number # into a file to read later in common.get_version_from_build() with open(os.path.join(target_dir, '0.version.txt'), 'w') as f: f.write(version) else: compile_version(version, target_dir, verbose=verbose) except urllib.error.URLError as e: msg = "Invalid version {}".format( version) if url is None else "Invalid url {}".format(url) msg = msg + " (underlying error is: {})".format(str(e)) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: {}".format( str(e))) except CCMError as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: raise CCMError( "Building C* version {} failed. Attempted to delete {} but failed. This will need to be manually deleted" .format(version, target_dir)) raise e
def enable_aoss(self): if self.version() < '6.0': common.error("Cannot enable AOSS in DSE clusters before 6.0") exit(1) self._misc_config_options['enable_aoss'] = True for node in self.nodelist(): port_offset = int(node.name[4:]) node.enable_aoss(thrift_port=10000 + port_offset, web_ui_port=9077 + port_offset) self._update_config()
def download_version(version, url=None, verbose=False, binary=False): """Download, extract, and build Cassandra tarball. if binary == True, download precompiled tarball, otherwise build from source tarball. """ assert_jdk_valid_for_cassandra_version(version) if binary: u = "%s/%s/apache-cassandra-%s-bin.tar.gz" % (ARCHIVE, version.split('-')[0], version) if url is None else url else: u = "%s/%s/apache-cassandra-%s-src.tar.gz" % (ARCHIVE, version.split('-')[0], version) if url is None else url _, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-") try: __download(u, target, show_progress=verbose) common.info("Extracting {} as version {} ...".format(target, version)) tar = tarfile.open(target) dir = next(tar).name.split("/")[0] tar.extractall(path=__get_dir()) tar.close() target_dir = os.path.join(__get_dir(), version) if os.path.exists(target_dir): rmdirs(target_dir) shutil.move(os.path.join(__get_dir(), dir), target_dir) if binary: # Binary installs don't have a build.xml that is needed # for pulling the version from. Write the version number # into a file to read later in common.get_version_from_build() with open(os.path.join(target_dir, '0.version.txt'), 'w') as f: f.write(version) else: compile_version(version, target_dir, verbose=verbose) except urllib.error.URLError as e: msg = "Invalid version {}" % version if url is None else "Invalid url {}".format(url) msg = msg + " (underlying error is: {})".format(str(e)) raise ArgumentError(msg) except tarfile.ReadError as e: raise ArgumentError("Unable to uncompress downloaded file: {}".format(str(e))) except CCMError as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: raise CCMError("Building C* version {} failed. Attempted to delete {} but failed. This will need to be manually deleted".format(version, target_dir)) raise e
def start(self, no_wait=False, verbose=False, wait_for_binary_proto=False, wait_other_notice=True, jvm_args=None, profile_options=None, quiet_start=False, allow_root=False): if jvm_args is None: jvm_args = [] common.assert_jdk_valid_for_cassandra_version(self.cassandra_version()) # check whether all loopback aliases are available before starting any nodes for node in list(self.nodes.values()): if not node.is_running(): for itf in node.network_interfaces.values(): if itf is not None: if not common.check_socket_available(itf, return_on_error=True): addr, port = itf common.error("Inet address {}:{} is not available; a cluster may already be running or you may need to add the loopback alias".format(addr, port)) sys.exit(1) started = [] for node in list(self.nodes.values()): if not node.is_running(): mark = 0 if os.path.exists(node.logfilename()): mark = node.mark_log() p = node.start(update_pid=False, jvm_args=jvm_args, profile_options=profile_options, verbose=verbose, quiet_start=quiet_start, allow_root=allow_root) # Prior to JDK8, starting every node at once could lead to a # nanotime collision where the RNG that generates a node's tokens # gives identical tokens to several nodes. Thus, we stagger # the node starts if common.get_jdk_version() < '1.8': time.sleep(1) started.append((node, p, mark)) if no_wait: time.sleep(2) # waiting 2 seconds to check for early errors and for the pid to be set else: for node, p, mark in started: try: start_message = "Listening for thrift clients..." if self.cassandra_version() < "2.2" else "Starting listening for CQL clients" node.watch_log_for(start_message, timeout=60, process=p, verbose=verbose, from_mark=mark) except RuntimeError: return None self.__update_pids(started) for node, p, _ in started: if not node.is_running(): raise NodeError("Error starting {0}.".format(node.name), p) if not no_wait: if wait_other_notice: for (node, _, mark), (other_node, _, _) in itertools.permutations(started, 2): node.watch_log_for_alive(other_node, from_mark=mark) if wait_for_binary_proto: for node, p, mark in started: node.wait_for_binary_interface(process=p, verbose=verbose, from_mark=mark) return started
def enable_aoss(self): common.error("Cannot enable AOSS in C* clusters") exit(1)
def clone_development(git_repo, version, verbose=False, alias=False, elassandra_version=None): print_(git_repo, version) target_dir = directory_name(version) assert target_dir if 'github' in version: git_repo_name, git_branch = github_username_and_branch_name(version) elif 'local:' in version: git_repo_name = 'local_{}'.format(git_repo) # add git repo location to distinguish cache location for differing repos git_branch = version.split(':')[-1] # last token on 'local:...' slugs should always be branch name elif alias: git_repo_name = 'alias_{}'.format(version.split('/')[0].split(':')[-1]) git_branch = version.split('/')[-1] else: git_repo_name = 'strapdata' git_branch = version.split(':', 1)[1] local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name) logfile = lastlogfilename() logger = get_logger(logfile) try: # Checkout/fetch a local repository cache to reduce the number of # remote fetches we need to perform: if not os.path.exists(local_git_cache): common.info("Cloning Elassandra...") process = subprocess.Popen( ['git', 'clone', '--mirror', git_repo, local_git_cache], cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git clone" else: common.info("Fetching Elassandra updates...") process = subprocess.Popen( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not update git" # Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. common.info("Cloning Elassandra (from local cache)") # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) process = subprocess.Popen( ['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git clone" else: process = subprocess.Popen( ['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git clone" # determine if the request is for a branch is_branch = False try: branch_listing = subprocess.check_output(['git', 'branch', '--all'], cwd=target_dir).decode('utf-8') branches = [b.strip() for b in branch_listing.replace('remotes/origin/', '').split()] is_branch = git_branch in branches except subprocess.CalledProcessError as cpe: common.error("Error Running Branch Filter: {}\nAssumming request is not for a branch".format(cpe.output)) # now check out the right version branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag' common.info("Checking out requested {} ({})".format(branch_or_sha_tag, git_branch)) if is_branch: # we use checkout -B with --track so we can specify that we want to track a specific branch # otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360 # we use -B instead of -b so we reset branches that already exist and create a new one otherwise process = subprocess.Popen(['git', 'checkout', '-B', git_branch, '--track', 'origin/{git_branch}'.format(git_branch=git_branch)], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) else: process = subprocess.Popen( ['git', 'checkout', git_branch], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) if int(out) != 0: raise CCMError('Could not check out git branch {branch}. ' 'Is this a valid branch name? (see {lastlog} or run ' '"ccm showlastlog" for details)'.format( branch=git_branch, lastlog=logfile )) # now compile compile_version(git_branch, target_dir, verbose, elassandra_version=elassandra_version) else: # branch is already checked out. See if it is behind and recompile if needed. process = subprocess.Popen( ['git', 'fetch', 'origin'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git fetch" process = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, status, _ = log_info(process, logger) if str(status).find('[behind') > -1: # If `status` looks like '## cassandra-2.2...origin/cassandra-2.2 [behind 9]\n' common.info("Branch is behind, recompiling") process = subprocess.Popen(['git', 'pull'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not do a git pull" process = subprocess.Popen([platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _, _ = log_info(process, logger) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose, elassandra_version=elassandra_version) elif re.search('\[.*?(ahead|behind).*?\]', status.decode("utf-8")) is not None: # status looks like '## trunk...origin/trunk [ahead 1, behind 29]\n' # If we have diverged in a way that fast-forward merging cannot solve, raise an exception so the cache is wiped common.error("Could not ascertain branch status, please resolve manually.") raise Exception else: # status looks like '## cassandra-2.2...origin/cassandra-2.2\n' common.debug("Branch up to date, not pulling.") except Exception as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: print_('Building C* version {version} failed. Attempted to delete {target_dir}' 'but failed. This will need to be manually deleted'.format( version=version, target_dir=target_dir )) finally: raise e
def clone_development(git_repo, version, verbose=False): print_(git_repo, version) target_dir = directory_name(version) assert target_dir if 'github' in version: git_repo_name, git_branch = github_username_and_branch_name(version) elif 'local:' in version: git_repo_name = 'local' git_branch = version.split(':', 1)[1] else: git_repo_name = 'apache' git_branch = version.split(':', 1)[1] local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name) logfile = lastlogfilename() with open(logfile, 'w') as lf: try: # Checkout/fetch a local repository cache to reduce the number of # remote fetches we need to perform: if not os.path.exists(local_git_cache): common.info("Cloning Cassandra...") out = subprocess.call( ['git', 'clone', '--mirror', git_repo, local_git_cache], cwd=__get_dir(), stdout=lf, stderr=lf) assert out == 0, "Could not do a git clone" else: common.info("Fetching Cassandra updates...") out = subprocess.call( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=lf, stderr=lf) # Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. common.info("Cloning Cassandra (from local cache)") # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) subprocess.call( ['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=lf, stderr=lf) else: subprocess.call( ['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=lf, stderr=lf) # determine if the request is for a branch is_branch = False try: branch_listing = subprocess.check_output( ['git', 'branch', '--all'], cwd=target_dir).decode('utf-8') branches = [ b.strip() for b in branch_listing.replace( 'remotes/origin/', '').split() ] is_branch = git_branch in branches except subprocess.CalledProcessError as cpe: common.error( "Error Running Branch Filter: {}\nAssumming request is not for a branch" .format(cpe.output)) # now check out the right version branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag' common.info("Checking out requested {} ({})".format( branch_or_sha_tag, git_branch)) if is_branch: # we use checkout -B with --track so we can specify that we want to track a specific branch # otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360 # we use -B instead of -b so we reset branches that already exist and create a new one otherwise out = subprocess.call([ 'git', 'checkout', '-B', git_branch, '--track', 'origin/{git_branch}'.format(git_branch=git_branch) ], cwd=target_dir, stdout=lf, stderr=lf) else: out = subprocess.call(['git', 'checkout', git_branch], cwd=target_dir, stdout=lf, stderr=lf) if int(out) != 0: raise CCMError( 'Could not check out git branch {branch}. ' 'Is this a valid branch name? (see {lastlog} or run ' '"ccm showlastlog" for details)'.format( branch=git_branch, lastlog=logfile)) # now compile compile_version(git_branch, target_dir, verbose) else: # branch is already checked out. See if it is behind and recompile if needed. out = subprocess.call(['git', 'fetch', 'origin'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git fetch" status = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=lf).communicate()[0] if str(status).find('[behind') > -1: common.info("Branch is behind, recompiling") out = subprocess.call(['git', 'pull'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git pull" out = subprocess.call( [platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose) except: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: raise CCMError( "Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted" % (version, target_dir)) raise
def clone_development(git_repo, version, verbose=False): print_(git_repo, version) target_dir = directory_name(version) assert target_dir if 'github' in version: git_repo_name, git_branch = github_username_and_branch_name(version) elif 'local:' in version: git_repo_name = 'local_{}'.format(git_repo) # add git repo location to distinguish cache location for differing repos git_branch = version.split(':')[-1] # last token on 'local:...' slugs should always be branch name else: git_repo_name = 'apache' git_branch = version.split(':', 1)[1] local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name) logfile = lastlogfilename() with open(logfile, 'w') as lf: try: # Checkout/fetch a local repository cache to reduce the number of # remote fetches we need to perform: if not os.path.exists(local_git_cache): common.info("Cloning Cassandra...") out = subprocess.call( ['git', 'clone', '--mirror', git_repo, local_git_cache], cwd=__get_dir(), stdout=lf, stderr=lf) assert out == 0, "Could not do a git clone" else: common.info("Fetching Cassandra updates...") out = subprocess.call( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=lf, stderr=lf) # Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. common.info("Cloning Cassandra (from local cache)") # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) subprocess.call(['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=lf, stderr=lf) else: subprocess.call(['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=lf, stderr=lf) # determine if the request is for a branch is_branch = False try: branch_listing = subprocess.check_output(['git', 'branch', '--all'], cwd=target_dir).decode('utf-8') branches = [b.strip() for b in branch_listing.replace('remotes/origin/', '').split()] is_branch = git_branch in branches except subprocess.CalledProcessError as cpe: common.error("Error Running Branch Filter: {}\nAssumming request is not for a branch".format(cpe.output)) # now check out the right version branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag' common.info("Checking out requested {} ({})".format(branch_or_sha_tag, git_branch)) if is_branch: # we use checkout -B with --track so we can specify that we want to track a specific branch # otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360 # we use -B instead of -b so we reset branches that already exist and create a new one otherwise out = subprocess.call(['git', 'checkout', '-B', git_branch, '--track', 'origin/{git_branch}'.format(git_branch=git_branch)], cwd=target_dir, stdout=lf, stderr=lf) else: out = subprocess.call(['git', 'checkout', git_branch], cwd=target_dir, stdout=lf, stderr=lf) if int(out) != 0: raise CCMError('Could not check out git branch {branch}. ' 'Is this a valid branch name? (see {lastlog} or run ' '"ccm showlastlog" for details)'.format( branch=git_branch, lastlog=logfile )) # now compile compile_version(git_branch, target_dir, verbose) else: # branch is already checked out. See if it is behind and recompile if needed. out = subprocess.call(['git', 'fetch', 'origin'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git fetch" status = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=lf).communicate()[0] if str(status).find('[behind') > -1: common.info("Branch is behind, recompiling") out = subprocess.call(['git', 'pull'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git pull" out = subprocess.call([platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose) except Exception as e: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: rmdirs(target_dir) common.error("Deleted {} due to error".format(target_dir)) except: print_('Building C* version {version} failed. Attempted to delete {target_dir}' 'but failed. This will need to be manually deleted'.format( version=version, target_dir=target_dir )) finally: raise e