def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]):
cdir = self.get_cassandra_dir()
cli = common.join_bin(cdir, 'bin', 'cqlsh')
env = common.make_cassandra_env(cdir, self.get_path())
host = self.network_interfaces['thrift'][0]
port = self.network_interfaces['thrift'][1]
args = cqlsh_options + [ host, str(port) ]
sys.stdout.flush()
if cmds is None:
os.execve(cli, [ common.platform_binary('cqlsh') ] + args, env)
else:
p = subprocess.Popen([ cli ] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
for cmd in cmds.split(';'):
p.stdin.write(cmd + ';\n')
p.stdin.write("quit;\n")
p.wait()
for err in p.stderr:
print_("(EE) ", err, end='')
if show_output:
i = 0
for log in p.stdout:
# first four lines are not interesting
if i >= 4:
print_(log, end='')
i = i + 1
def get_stress_bin(cassandra_dir):
candidates = [
os.path.join(cassandra_dir, 'contrib', 'stress', 'bin', 'stress'),
os.path.join(cassandra_dir, 'tools', 'stress', 'bin', 'stress'),
os.path.join(cassandra_dir, 'tools', 'bin', 'stress'),
os.path.join(cassandra_dir, 'tools', 'bin', 'cassandra-stress')
]
candidates = [common.platform_binary(s) for s in candidates]
for candidate in candidates:
if os.path.exists(candidate):
stress = candidate
break
else:
raise Exception("Cannot find stress binary (maybe it isn't compiled)")
# make sure it's executable -> win32 doesn't care
if sys.platform == "cygwin":
# Yes, we're unwinding the path join from above.
path = parse_path(stress)
short_bin = parse_bin(stress)
add_exec_permission(path, short_bin)
elif not os.access(stress, os.X_OK):
try:
# try to add user execute permissions
# os.chmod doesn't work on Windows and isn't necessary unless in cygwin...
if sys.platform == "cygwin":
common.add_exec_permission(path, stress)
else:
os.chmod(stress, os.stat(stress).st_mode | stat.S_IXUSR)
except:
raise Exception("stress binary is not executable: %s" % (stress,))
return stress
def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]):
cdir = self.get_cassandra_dir()
cli = common.join_bin(cdir, 'bin', 'cqlsh')
env = common.make_cassandra_env(cdir, self.get_path())
host = self.network_interfaces['thrift'][0]
if self.cluster.version() >= "2.1":
port = self.network_interfaces['binary'][1]
else:
port = self.network_interfaces['thrift'][1]
args = cqlsh_options + [host, str(port)]
sys.stdout.flush()
if cmds is None:
os.execve(cli, [common.platform_binary('cqlsh')] + args, env)
else:
p = subprocess.Popen([cli] + args,
env=env,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
for cmd in cmds.split(';'):
p.stdin.write(cmd + ';\n')
p.stdin.write("quit;\n")
p.wait()
for err in p.stderr:
print "(EE) " + err,
if show_output:
i = 0
for log in p.stdout:
# first four lines are not interesting
if i >= 4:
print log,
i = i + 1
def get_stress_bin(cassandra_dir):
candidates = [
os.path.join(cassandra_dir, 'contrib', 'stress', 'bin', 'stress'),
os.path.join(cassandra_dir, 'tools', 'stress', 'bin', 'stress'),
os.path.join(cassandra_dir, 'tools', 'bin', 'stress'),
os.path.join(cassandra_dir, 'tools', 'bin', 'cassandra-stress')
]
candidates = [common.platform_binary(s) for s in candidates]
for candidate in candidates:
if os.path.exists(candidate):
stress = candidate
break
else:
raise Exception("Cannot find stress binary (maybe it isn't compiled)")
# make sure it's executable -> win32 doesn't care
if sys.platform == "cygwin":
# Yes, we're unwinding the path join from above.
path = parse_path(stress)
short_bin = parse_bin(stress)
add_exec_permission(path, short_bin)
elif not os.access(stress, os.X_OK):
try:
# try to add user execute permissions
# os.chmod doesn't work on Windows and isn't necessary unless in cygwin...
if sys.platform == "cygwin":
common.add_exec_permission(path, stress)
else:
os.chmod(stress, os.stat(stress).st_mode | stat.S_IXUSR)
except:
raise Exception("stress binary is not executable: %s" % (stress, ))
return stress
def compile_version(version, target_dir, verbose=False):
# compiling cassandra and the stress tool
logfile = os.path.join(__get_dir(), "last.log")
if verbose:
print "Compiling Cassandra %s ..." % version
with open(logfile, 'w') as lf:
lf.write("--- Cassandra build -------------------\n")
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
while attempt < 3 and ret_val is not 0:
if attempt > 0:
lf.write("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt)
ret_val = subprocess.call([common.platform_binary('ant'),'jar'], cwd=target_dir, stdout=lf, stderr=lf)
attempt += 1
if ret_val is not 0:
raise common.CCMError("Error compiling Cassandra. See %s for details" % logfile)
except OSError, e:
raise common.CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile)
lf.write("\n\n--- cassandra/stress build ------------\n")
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
if subprocess.call([common.platform_binary('ant'), 'build'], cwd=stress_dir, stdout=lf, stderr=lf) is not 0:
if subprocess.call([common.platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=lf, stderr=lf) is not 0:
raise common.CCMError("Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise common.CCMError("Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)" % str(e))
def load(self, options):
for itf in self.network_interfaces.values():
if itf:
common.check_socket_available(itf)
cdir = self.get_cassandra_dir()
loader_bin = common.join_bin(cdir, 'bin', 'sstableloader')
env = common.make_cassandra_env(cdir, self.get_path())
if not "-d" in options:
l = [ node.network_interfaces['storage'][0] for node in self.cluster.nodes.values() if node.is_live() ]
options = [ "-d", ",".join(l) ] + options
#print "Executing with", options
os.execve(loader_bin, [ common.platform_binary('sstableloader') ] + options, env)
def load(self, options):
for itf in self.network_interfaces.values():
if itf:
common.check_socket_available(itf)
cdir = self.get_cassandra_dir()
loader_bin = common.join_bin(cdir, 'bin', 'sstableloader')
env = common.make_cassandra_env(cdir, self.get_path())
if not "-d" in options:
l = [
node.network_interfaces['storage'][0]
for node in self.cluster.nodes.values() if node.is_live()
]
options = ["-d", ",".join(l)] + options
#print "Executing with", options
os.execve(loader_bin,
[common.platform_binary('sstableloader')] + options, env)
def scrub(self, options):
cdir = self.get_cassandra_dir()
scrub_bin = common.join_bin(cdir, 'bin', 'sstablescrub')
env = common.make_cassandra_env(cdir, self.get_path())
os.execve(scrub_bin, [ common.platform_binary('sstablescrub') ] + options, env)
def clone_development(version, verbose=False):
local_git_cache = os.path.join(__get_dir(), '_git_cache')
target_dir = os.path.join(__get_dir(), version.replace(
':', '_')) # handle git branches like 'git:trunk'.
git_branch = version[4:] # the part of the version after the 'git:'
logfile = os.path.join(__get_dir(), "last.log")
with open(logfile, 'w') as lf:
try:
#Checkout/fetch a local repository cache to reduce the number of
#remote fetches we need to perform:
if not os.path.exists(local_git_cache):
if verbose:
print "Cloning Cassandra..."
out = subprocess.call(
['git', 'clone', '--mirror', GIT_REPO, local_git_cache],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git clone"
else:
if verbose:
print "Fetching Cassandra updates..."
out = subprocess.call(
['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'],
cwd=local_git_cache,
stdout=lf,
stderr=lf)
#Checkout the version we want from the local cache:
if not os.path.exists(target_dir):
# development branch doesn't exist. Check it out.
if verbose:
print "Cloning Cassandra (from local cache)"
# git on cygwin appears to be adding `cwd` to the commands which is breaking clone
if sys.platform == "cygwin":
local_split = local_git_cache.split(os.sep)
target_split = target_dir.split(os.sep)
subprocess.call(
['git', 'clone', local_split[-1], target_split[-1]],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
else:
subprocess.call(
['git', 'clone', local_git_cache, target_dir],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
# now check out the right version
if verbose:
print "Checking out requested branch (%s)" % git_branch
out = subprocess.call(['git', 'checkout', git_branch],
cwd=target_dir,
stdout=lf,
stderr=lf)
if int(out) != 0:
raise common.CCMError(
"Could not check out git branch %s. Is this a valid branch name? (see last.log for details)"
% git_branch)
# now compile
compile_version(git_branch, target_dir, verbose)
else: # branch is already checked out. See if it is behind and recompile if needed.
out = subprocess.call(['git', 'fetch', 'origin'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git fetch"
status = subprocess.Popen(['git', 'status', '-sb'],
cwd=target_dir,
stdout=subprocess.PIPE,
stderr=lf).communicate()[0]
if status.find('[behind') > -1:
if verbose:
print "Branch is behind, recompiling"
out = subprocess.call(['git', 'pull'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git pull"
out = subprocess.call(
[common.platform_binary('ant'), 'realclean'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not run 'ant realclean'"
# now compile
compile_version(git_branch, target_dir, verbose)
except:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
shutil.rmtree(target_dir)
except:
pass
raise
def compile_version(version, target_dir, verbose=False):
# compiling cassandra and the stress tool
logfile = os.path.join(__get_dir(), "last.log")
if verbose:
print "Compiling Cassandra %s ..." % version
with open(logfile, 'w') as lf:
lf.write("--- Cassandra build -------------------\n")
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
while attempt < 3 and ret_val is not 0:
if attempt > 0:
lf.write("\n\n`ant jar` failed. Retry #%s...\n\n" %
attempt)
ret_val = subprocess.call(
[common.platform_binary('ant'), 'jar'],
cwd=target_dir,
stdout=lf,
stderr=lf)
attempt += 1
if ret_val is not 0:
raise common.CCMError(
"Error compiling Cassandra. See %s for details" % logfile)
except OSError, e:
raise common.CCMError(
"Error compiling Cassandra. Is ant installed? See %s for details"
% logfile)
lf.write("\n\n--- cassandra/stress build ------------\n")
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(
build_xml
): # building stress separately is only necessary pre-1.1
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(
full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH
| stat.S_IXOTH)
if subprocess.call([common.platform_binary('ant'), 'build'],
cwd=stress_dir,
stdout=lf,
stderr=lf) is not 0:
if subprocess.call(
[common.platform_binary('ant'), 'stress-build'],
cwd=target_dir,
stdout=lf,
stderr=lf) is not 0:
raise common.CCMError(
"Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise common.CCMError(
"Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)"
% str(e))
def scrub(self, options):
cdir = self.get_cassandra_dir()
scrub_bin = common.join_bin(cdir, 'bin', 'sstablescrub')
env = common.make_cassandra_env(cdir, self.get_path())
os.execve(scrub_bin,
[common.platform_binary('sstablescrub')] + options, env)
def clone_development(version, verbose=False):
local_git_cache = os.path.join(__get_dir(), '_git_cache')
target_dir = os.path.join(__get_dir(), version.replace(':', '_')) # handle git branches like 'git:trunk'.
git_branch = version[4:] # the part of the version after the 'git:'
logfile = os.path.join(__get_dir(), "last.log")
with open(logfile, 'w') as lf:
try:
#Checkout/fetch a local repository cache to reduce the number of
#remote fetches we need to perform:
if not os.path.exists(local_git_cache):
if verbose:
print "Cloning Cassandra..."
out = subprocess.call(
['git', 'clone', '--mirror', GIT_REPO, local_git_cache],
cwd=__get_dir(), stdout=lf, stderr=lf)
assert out == 0, "Could not do a git clone"
else:
if verbose:
print "Fetching Cassandra updates..."
out = subprocess.call(
['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'],
cwd=local_git_cache, stdout=lf, stderr=lf)
#Checkout the version we want from the local cache:
if not os.path.exists(target_dir):
# development branch doesn't exist. Check it out.
if verbose:
print "Cloning Cassandra (from local cache)"
# git on cygwin appears to be adding `cwd` to the commands which is breaking clone
if sys.platform == "cygwin":
local_split = local_git_cache.split(os.sep)
target_split = target_dir.split(os.sep)
subprocess.call(['git', 'clone', local_split[-1], target_split[-1]], cwd=__get_dir(), stdout=lf, stderr=lf)
else:
subprocess.call(['git', 'clone', local_git_cache, target_dir], cwd=__get_dir(), stdout=lf, stderr=lf)
# now check out the right version
if verbose:
print "Checking out requested branch (%s)" % git_branch
out = subprocess.call(['git', 'checkout', git_branch], cwd=target_dir, stdout=lf, stderr=lf)
if int(out) != 0:
raise common.CCMError("Could not check out git branch %s. Is this a valid branch name? (see last.log for details)" % git_branch)
# now compile
compile_version(git_branch, target_dir, verbose)
else: # branch is already checked out. See if it is behind and recompile if needed.
out = subprocess.call(['git', 'fetch', 'origin'], cwd=target_dir, stdout=lf, stderr=lf)
assert out == 0, "Could not do a git fetch"
status = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=lf).communicate()[0]
if status.find('[behind') > -1:
if verbose:
print "Branch is behind, recompiling"
out = subprocess.call(['git', 'pull'], cwd=target_dir, stdout=lf, stderr=lf)
assert out == 0, "Could not do a git pull"
out = subprocess.call([common.platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=lf, stderr=lf)
assert out == 0, "Could not run 'ant realclean'"
# now compile
compile_version(git_branch, target_dir, verbose)
except:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
shutil.rmtree(target_dir)
except: pass
raise
