def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cqlsh') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] port = self.network_interfaces['thrift'][1] args = cqlsh_options + [ host, str(port) ] sys.stdout.flush() if cmds is None: os.execve(cli, [ common.platform_binary('cqlsh') ] + args, env) else: p = subprocess.Popen([ cli ] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) for cmd in cmds.split(';'): p.stdin.write(cmd + ';\n') p.stdin.write("quit;\n") p.wait() for err in p.stderr: print_("(EE) ", err, end='') if show_output: i = 0 for log in p.stdout: # first four lines are not interesting if i >= 4: print_(log, end='') i = i + 1
def get_stress_bin(cassandra_dir): candidates = [ os.path.join(cassandra_dir, 'contrib', 'stress', 'bin', 'stress'), os.path.join(cassandra_dir, 'tools', 'stress', 'bin', 'stress'), os.path.join(cassandra_dir, 'tools', 'bin', 'stress'), os.path.join(cassandra_dir, 'tools', 'bin', 'cassandra-stress') ] candidates = [common.platform_binary(s) for s in candidates] for candidate in candidates: if os.path.exists(candidate): stress = candidate break else: raise Exception("Cannot find stress binary (maybe it isn't compiled)") # make sure it's executable -> win32 doesn't care if sys.platform == "cygwin": # Yes, we're unwinding the path join from above. path = parse_path(stress) short_bin = parse_bin(stress) add_exec_permission(path, short_bin) elif not os.access(stress, os.X_OK): try: # try to add user execute permissions # os.chmod doesn't work on Windows and isn't necessary unless in cygwin... if sys.platform == "cygwin": common.add_exec_permission(path, stress) else: os.chmod(stress, os.stat(stress).st_mode | stat.S_IXUSR) except: raise Exception("stress binary is not executable: %s" % (stress,)) return stress
def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cqlsh') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] if self.cluster.version() >= "2.1": port = self.network_interfaces['binary'][1] else: port = self.network_interfaces['thrift'][1] args = cqlsh_options + [host, str(port)] sys.stdout.flush() if cmds is None: os.execve(cli, [common.platform_binary('cqlsh')] + args, env) else: p = subprocess.Popen([cli] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) for cmd in cmds.split(';'): p.stdin.write(cmd + ';\n') p.stdin.write("quit;\n") p.wait() for err in p.stderr: print "(EE) " + err, if show_output: i = 0 for log in p.stdout: # first four lines are not interesting if i >= 4: print log, i = i + 1
def get_stress_bin(cassandra_dir): candidates = [ os.path.join(cassandra_dir, 'contrib', 'stress', 'bin', 'stress'), os.path.join(cassandra_dir, 'tools', 'stress', 'bin', 'stress'), os.path.join(cassandra_dir, 'tools', 'bin', 'stress'), os.path.join(cassandra_dir, 'tools', 'bin', 'cassandra-stress') ] candidates = [common.platform_binary(s) for s in candidates] for candidate in candidates: if os.path.exists(candidate): stress = candidate break else: raise Exception("Cannot find stress binary (maybe it isn't compiled)") # make sure it's executable -> win32 doesn't care if sys.platform == "cygwin": # Yes, we're unwinding the path join from above. path = parse_path(stress) short_bin = parse_bin(stress) add_exec_permission(path, short_bin) elif not os.access(stress, os.X_OK): try: # try to add user execute permissions # os.chmod doesn't work on Windows and isn't necessary unless in cygwin... if sys.platform == "cygwin": common.add_exec_permission(path, stress) else: os.chmod(stress, os.stat(stress).st_mode | stat.S_IXUSR) except: raise Exception("stress binary is not executable: %s" % (stress, )) return stress
def compile_version(version, target_dir, verbose=False): # compiling cassandra and the stress tool logfile = os.path.join(__get_dir(), "last.log") if verbose: print "Compiling Cassandra %s ..." % version with open(logfile, 'w') as lf: lf.write("--- Cassandra build -------------------\n") try: # Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543 # Similar patch seen with buildbot attempt = 0 ret_val = 1 while attempt < 3 and ret_val is not 0: if attempt > 0: lf.write("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt) ret_val = subprocess.call([common.platform_binary('ant'),'jar'], cwd=target_dir, stdout=lf, stderr=lf) attempt += 1 if ret_val is not 0: raise common.CCMError("Error compiling Cassandra. See %s for details" % logfile) except OSError, e: raise common.CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile) lf.write("\n\n--- cassandra/stress build ------------\n") stress_dir = os.path.join(target_dir, "tools", "stress") if ( version >= "0.8.0") else \ os.path.join(target_dir, "contrib", "stress") build_xml = os.path.join(stress_dir, 'build.xml') if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1 try: # set permissions correctly, seems to not always be the case stress_bin_dir = os.path.join(stress_dir, 'bin') for f in os.listdir(stress_bin_dir): full_path = os.path.join(stress_bin_dir, f) os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) if subprocess.call([common.platform_binary('ant'), 'build'], cwd=stress_dir, stdout=lf, stderr=lf) is not 0: if subprocess.call([common.platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=lf, stderr=lf) is not 0: raise common.CCMError("Error compiling Cassandra stress tool. " "See %s for details (you will still be able to use ccm " "but not the stress related commands)" % logfile) except IOError as e: raise common.CCMError("Error compiling Cassandra stress tool: %s (you will " "still be able to use ccm but not the stress related commands)" % str(e))
def load(self, options): for itf in self.network_interfaces.values(): if itf: common.check_socket_available(itf) cdir = self.get_cassandra_dir() loader_bin = common.join_bin(cdir, 'bin', 'sstableloader') env = common.make_cassandra_env(cdir, self.get_path()) if not "-d" in options: l = [ node.network_interfaces['storage'][0] for node in self.cluster.nodes.values() if node.is_live() ] options = [ "-d", ",".join(l) ] + options #print "Executing with", options os.execve(loader_bin, [ common.platform_binary('sstableloader') ] + options, env)
def load(self, options): for itf in self.network_interfaces.values(): if itf: common.check_socket_available(itf) cdir = self.get_cassandra_dir() loader_bin = common.join_bin(cdir, 'bin', 'sstableloader') env = common.make_cassandra_env(cdir, self.get_path()) if not "-d" in options: l = [ node.network_interfaces['storage'][0] for node in self.cluster.nodes.values() if node.is_live() ] options = ["-d", ",".join(l)] + options #print "Executing with", options os.execve(loader_bin, [common.platform_binary('sstableloader')] + options, env)
def scrub(self, options): cdir = self.get_cassandra_dir() scrub_bin = common.join_bin(cdir, 'bin', 'sstablescrub') env = common.make_cassandra_env(cdir, self.get_path()) os.execve(scrub_bin, [ common.platform_binary('sstablescrub') ] + options, env)
def clone_development(version, verbose=False): local_git_cache = os.path.join(__get_dir(), '_git_cache') target_dir = os.path.join(__get_dir(), version.replace( ':', '_')) # handle git branches like 'git:trunk'. git_branch = version[4:] # the part of the version after the 'git:' logfile = os.path.join(__get_dir(), "last.log") with open(logfile, 'w') as lf: try: #Checkout/fetch a local repository cache to reduce the number of #remote fetches we need to perform: if not os.path.exists(local_git_cache): if verbose: print "Cloning Cassandra..." out = subprocess.call( ['git', 'clone', '--mirror', GIT_REPO, local_git_cache], cwd=__get_dir(), stdout=lf, stderr=lf) assert out == 0, "Could not do a git clone" else: if verbose: print "Fetching Cassandra updates..." out = subprocess.call( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=lf, stderr=lf) #Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. if verbose: print "Cloning Cassandra (from local cache)" # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) subprocess.call( ['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=lf, stderr=lf) else: subprocess.call( ['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=lf, stderr=lf) # now check out the right version if verbose: print "Checking out requested branch (%s)" % git_branch out = subprocess.call(['git', 'checkout', git_branch], cwd=target_dir, stdout=lf, stderr=lf) if int(out) != 0: raise common.CCMError( "Could not check out git branch %s. Is this a valid branch name? (see last.log for details)" % git_branch) # now compile compile_version(git_branch, target_dir, verbose) else: # branch is already checked out. See if it is behind and recompile if needed. out = subprocess.call(['git', 'fetch', 'origin'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git fetch" status = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=lf).communicate()[0] if status.find('[behind') > -1: if verbose: print "Branch is behind, recompiling" out = subprocess.call(['git', 'pull'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git pull" out = subprocess.call( [common.platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose) except: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: shutil.rmtree(target_dir) except: pass raise
def compile_version(version, target_dir, verbose=False): # compiling cassandra and the stress tool logfile = os.path.join(__get_dir(), "last.log") if verbose: print "Compiling Cassandra %s ..." % version with open(logfile, 'w') as lf: lf.write("--- Cassandra build -------------------\n") try: # Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543 # Similar patch seen with buildbot attempt = 0 ret_val = 1 while attempt < 3 and ret_val is not 0: if attempt > 0: lf.write("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt) ret_val = subprocess.call( [common.platform_binary('ant'), 'jar'], cwd=target_dir, stdout=lf, stderr=lf) attempt += 1 if ret_val is not 0: raise common.CCMError( "Error compiling Cassandra. See %s for details" % logfile) except OSError, e: raise common.CCMError( "Error compiling Cassandra. Is ant installed? See %s for details" % logfile) lf.write("\n\n--- cassandra/stress build ------------\n") stress_dir = os.path.join(target_dir, "tools", "stress") if ( version >= "0.8.0") else \ os.path.join(target_dir, "contrib", "stress") build_xml = os.path.join(stress_dir, 'build.xml') if os.path.exists( build_xml ): # building stress separately is only necessary pre-1.1 try: # set permissions correctly, seems to not always be the case stress_bin_dir = os.path.join(stress_dir, 'bin') for f in os.listdir(stress_bin_dir): full_path = os.path.join(stress_bin_dir, f) os.chmod( full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) if subprocess.call([common.platform_binary('ant'), 'build'], cwd=stress_dir, stdout=lf, stderr=lf) is not 0: if subprocess.call( [common.platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=lf, stderr=lf) is not 0: raise common.CCMError( "Error compiling Cassandra stress tool. " "See %s for details (you will still be able to use ccm " "but not the stress related commands)" % logfile) except IOError as e: raise common.CCMError( "Error compiling Cassandra stress tool: %s (you will " "still be able to use ccm but not the stress related commands)" % str(e))
def scrub(self, options): cdir = self.get_cassandra_dir() scrub_bin = common.join_bin(cdir, 'bin', 'sstablescrub') env = common.make_cassandra_env(cdir, self.get_path()) os.execve(scrub_bin, [common.platform_binary('sstablescrub')] + options, env)
def clone_development(version, verbose=False): local_git_cache = os.path.join(__get_dir(), '_git_cache') target_dir = os.path.join(__get_dir(), version.replace(':', '_')) # handle git branches like 'git:trunk'. git_branch = version[4:] # the part of the version after the 'git:' logfile = os.path.join(__get_dir(), "last.log") with open(logfile, 'w') as lf: try: #Checkout/fetch a local repository cache to reduce the number of #remote fetches we need to perform: if not os.path.exists(local_git_cache): if verbose: print "Cloning Cassandra..." out = subprocess.call( ['git', 'clone', '--mirror', GIT_REPO, local_git_cache], cwd=__get_dir(), stdout=lf, stderr=lf) assert out == 0, "Could not do a git clone" else: if verbose: print "Fetching Cassandra updates..." out = subprocess.call( ['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'], cwd=local_git_cache, stdout=lf, stderr=lf) #Checkout the version we want from the local cache: if not os.path.exists(target_dir): # development branch doesn't exist. Check it out. if verbose: print "Cloning Cassandra (from local cache)" # git on cygwin appears to be adding `cwd` to the commands which is breaking clone if sys.platform == "cygwin": local_split = local_git_cache.split(os.sep) target_split = target_dir.split(os.sep) subprocess.call(['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=lf, stderr=lf) else: subprocess.call(['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=lf, stderr=lf) # now check out the right version if verbose: print "Checking out requested branch (%s)" % git_branch out = subprocess.call(['git', 'checkout', git_branch], cwd=target_dir, stdout=lf, stderr=lf) if int(out) != 0: raise common.CCMError("Could not check out git branch %s. Is this a valid branch name? (see last.log for details)" % git_branch) # now compile compile_version(git_branch, target_dir, verbose) else: # branch is already checked out. See if it is behind and recompile if needed. out = subprocess.call(['git', 'fetch', 'origin'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git fetch" status = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=lf).communicate()[0] if status.find('[behind') > -1: if verbose: print "Branch is behind, recompiling" out = subprocess.call(['git', 'pull'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not do a git pull" out = subprocess.call([common.platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=lf, stderr=lf) assert out == 0, "Could not run 'ant realclean'" # now compile compile_version(git_branch, target_dir, verbose) except: # wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs. try: shutil.rmtree(target_dir) except: pass raise