def compile_version(version, target_dir, verbose=False):
# compiling cassandra and the stress tool
logfile = os.path.join(__get_dir(), "last.log")
if verbose:
print "Compiling Cassandra %s ..." % version
with open(logfile, 'w') as lf:
lf.write("--- Cassandra build -------------------\n")
try:
if subprocess.call(['ant', 'jar'],
cwd=target_dir,
stdout=lf,
stderr=lf) is not 0:
raise common.CCMError(
"Error compiling Cassandra. See %s for details" % logfile)
except OSError, e:
raise common.CCMError(
"Error compiling Cassandra. Is ant installed? See %s for details"
% logfile)
lf.write("\n\n--- cassandra/stress build ------------\n")
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(
build_xml
): # building stress separately is only necessary pre-1.1
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(
full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH
| stat.S_IXOTH)
if subprocess.call(['ant', 'build'],
cwd=stress_dir,
stdout=lf,
stderr=lf) is not 0:
if subprocess.call(['ant', 'stress-build'],
cwd=target_dir,
stdout=lf,
stderr=lf) is not 0:
raise common.CCMError(
"Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise common.CCMError(
"Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)"
% str(e))
def __get_version_from_build(self):
cassandra_dir = self.get_cassandra_dir()
build = os.path.join(cassandra_dir, 'build.xml')
with open(build) as f:
for line in f:
match = re.search('name="base\.version" value="([0-9.]+)[^"]*"', line)
if match:
return match.group(1)
raise common.CCMError("Cannot find version")
def __download(url, target, show_progress=False):
u = urllib2.urlopen(url)
f = open(target, 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
if show_progress:
print "Downloading %s to %s (%.3fMB)" % (url, target,
float(file_size) /
(1024 * 1024))
file_size_dl = 0
block_sz = 8192
status = None
attempts = 0
while file_size_dl < file_size:
buffer = u.read(block_sz)
if not buffer:
attempts = attempts + 1
if attempts >= 5:
raise common.CCMError(
"Error downloading file (nothing read after %i attemps, downloded only %i of %i bytes)"
% (attempts, file_size_dl, file_size))
time.sleep(0.5 * attempts)
continue
else:
attemps = 0
file_size_dl += len(buffer)
f.write(buffer)
if show_progress:
status = r"%10d [%3.2f%%]" % (file_size_dl,
file_size_dl * 100. / file_size)
status = chr(8) * (len(status) + 1) + status
print status,
if show_progress:
print ""
f.close()
u.close()
def clone_development(version, verbose=False):
local_git_cache = os.path.join(__get_dir(), '_git_cache')
target_dir = os.path.join(__get_dir(), version.replace(
':', '_')) # handle git branches like 'git:trunk'.
git_branch = version[4:] # the part of the version after the 'git:'
logfile = os.path.join(__get_dir(), "last.log")
with open(logfile, 'w') as lf:
try:
#Checkout/fetch a local repository cache to reduce the number of
#remote fetches we need to perform:
if not os.path.exists(local_git_cache):
if verbose:
print "Cloning Cassandra..."
out = subprocess.call(
['git', 'clone', '--mirror', GIT_REPO, local_git_cache],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git clone"
else:
if verbose:
print "Fetching Cassandra updates..."
out = subprocess.call(
['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'],
cwd=local_git_cache,
stdout=lf,
stderr=lf)
#Checkout the version we want from the local cache:
if not os.path.exists(target_dir):
# development branch doesn't exist. Check it out.
if verbose:
print "Cloning Cassandra (from local cache)"
# git on cygwin appears to be adding `cwd` to the commands which is breaking clone
if sys.platform == "cygwin":
local_split = local_git_cache.split(os.sep)
target_split = target_dir.split(os.sep)
subprocess.call(
['git', 'clone', local_split[-1], target_split[-1]],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
else:
subprocess.call(
['git', 'clone', local_git_cache, target_dir],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
# now check out the right version
if verbose:
print "Checking out requested branch (%s)" % git_branch
out = subprocess.call(['git', 'checkout', git_branch],
cwd=target_dir,
stdout=lf,
stderr=lf)
if int(out) != 0:
raise common.CCMError(
"Could not check out git branch %s. Is this a valid branch name? (see last.log for details)"
% git_branch)
# now compile
compile_version(git_branch, target_dir, verbose)
else: # branch is already checked out. See if it is behind and recompile if needed.
out = subprocess.call(['git', 'fetch', 'origin'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git fetch"
status = subprocess.Popen(['git', 'status', '-sb'],
cwd=target_dir,
stdout=subprocess.PIPE,
stderr=lf).communicate()[0]
if status.find('[behind') > -1:
if verbose:
print "Branch is behind, recompiling"
out = subprocess.call(['git', 'pull'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git pull"
out = subprocess.call(
[common.platform_binary('ant'), 'realclean'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not run 'ant realclean'"
# now compile
compile_version(git_branch, target_dir, verbose)
except:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
shutil.rmtree(target_dir)
except:
pass
raise
def compile_version(version, target_dir, verbose=False):
# compiling cassandra and the stress tool
logfile = os.path.join(__get_dir(), "last.log")
if verbose:
print "Compiling Cassandra %s ..." % version
with open(logfile, 'w') as lf:
lf.write("--- Cassandra build -------------------\n")
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
while attempt < 3 and ret_val is not 0:
if attempt > 0:
lf.write("\n\n`ant jar` failed. Retry #%s...\n\n" %
attempt)
ret_val = subprocess.call(
[common.platform_binary('ant'), 'jar'],
cwd=target_dir,
stdout=lf,
stderr=lf)
attempt += 1
if ret_val is not 0:
raise common.CCMError(
"Error compiling Cassandra. See %s for details" % logfile)
except OSError, e:
raise common.CCMError(
"Error compiling Cassandra. Is ant installed? See %s for details"
% logfile)
lf.write("\n\n--- cassandra/stress build ------------\n")
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(
build_xml
): # building stress separately is only necessary pre-1.1
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(
full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH
| stat.S_IXOTH)
if subprocess.call([common.platform_binary('ant'), 'build'],
cwd=stress_dir,
stdout=lf,
stderr=lf) is not 0:
if subprocess.call(
[common.platform_binary('ant'), 'stress-build'],
cwd=target_dir,
stdout=lf,
stderr=lf) is not 0:
raise common.CCMError(
"Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise common.CCMError(
"Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)"
% str(e))