def compile_version_elassandra(version, target_dir, verbose=False, elassandra_version=None):
assert_jdk_valid_for_cassandra_version(get_version_from_build(target_dir))
# compiling cassandra and the stress tool
logfile = lastlogfilename()
logger = get_logger(logfile)
common.info("Compiling Elassandra {} ...".format(version))
logger.info("--- Elassandra Build -------------------\n")
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
if elassandra_version is None:
logger.info("elassandra version is not set, trying with 2.4.2")
elassandra_version = "2.4.2"
targz_file = "distribution/tar/target/releases/elassandra-%s.tar.gz" % elassandra_version
target_install_dir = os.path.join(target_dir, "elassandra-%s")
while attempt < 3 and ret_val is not 0:
if attempt > 0:
logger.info("\n\n`mvn package -DskipTests` failed. Retry #%s...\n\n" % attempt)
process = subprocess.Popen([platform_binary('mvn'), 'package', '-DskipTests'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val is 0:
process = subprocess.Popen([platform_binary('tar'), '-xzf', targz_file], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
attempt += 1
if ret_val is not 0:
raise CCMError('Error compiling Elassandra. See {logfile} or run '
'"ccm showlastlog" for details'.format(logfile=logfile))
except OSError as e:
raise CCMError("Error compiling Elassandra. Is maven installed? See %s for details" % logfile)
def compile_version(version, target_dir, verbose=False):
assert_jdk_valid_for_cassandra_version(get_version_from_build(target_dir))
# compiling cassandra and the stress tool
logfile = lastlogfilename()
logger = get_logger(logfile)
common.info("Compiling Cassandra {} ...".format(version))
logger.info("--- Cassandra Build -------------------\n")
default_build_properties = os.path.join(common.get_default_path(), 'build.properties.default')
if os.path.exists(default_build_properties):
target_build_properties = os.path.join(target_dir, 'build.properties')
logger.info("Copying %s to %s\n" % (default_build_properties, target_build_properties))
shutil.copyfile(default_build_properties, target_build_properties)
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
while attempt < 3 and ret_val is not 0:
if attempt > 0:
logger.info("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt)
process = subprocess.Popen([platform_binary('ant'), 'jar'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
attempt += 1
if ret_val is not 0:
raise CCMError('Error compiling Cassandra. See {logfile} or run '
'"ccm showlastlog" for details'.format(logfile=logfile))
except OSError as e:
raise CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile)
logger.info("\n\n--- cassandra/stress build ------------\n")
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
process = subprocess.Popen([platform_binary('ant'), 'build'], cwd=stress_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val is not 0:
process = subprocess.Popen([platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val is not 0:
raise CCMError("Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise CCMError("Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)" % str(e))
def get_relocatable_s3_url(branch, s3_version, links):
for reloc_url in links:
s3_url = reloc_url.format(branch, s3_version)
resp = aws_bucket_ls(s3_url)
if resp:
return s3_url
raise CCMError(f"s3 url was not found for {branch}:{s3_version}")
def download_version(version, url=None, verbose=False, binary=False):
"""Download, extract, and build Cassandra tarball.
if binary == True, download precompiled tarball, otherwise build from source tarball.
"""
assert_jdk_valid_for_cassandra_version(version)
archive_url = ARCHIVE
if CCM_CONFIG.has_option('repositories', 'cassandra'):
archive_url = CCM_CONFIG.get('repositories', 'cassandra')
if binary:
archive_url = "%s/%s/apache-cassandra-%s-bin.tar.gz" % (
archive_url, version.split('-')[0],
version) if url is None else url
else:
archive_url = "%s/%s/apache-cassandra-%s-src.tar.gz" % (
archive_url, version.split('-')[0],
version) if url is None else url
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
__download(archive_url, target, show_progress=verbose)
common.info("Extracting {} as version {} ...".format(target, version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0] # pylint: disable=all
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), version)
if os.path.exists(target_dir):
rmdirs(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
if binary:
# Binary installs don't have a build.xml that is needed
# for pulling the version from. Write the version number
# into a file to read later in common.get_version_from_build()
with open(os.path.join(target_dir, '0.version.txt'), 'w') as f:
f.write(version)
else:
compile_version(version, target_dir, verbose=verbose)
except urllib.error.URLError as e:
msg = "Invalid version {}".format(
version) if url is None else "Invalid url {}".format(url)
msg = msg + " (underlying error is: {})".format(str(e))
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: {}".format(
str(e)))
except CCMError as e:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
rmdirs(target_dir)
common.error("Deleted {} due to error".format(target_dir))
except:
raise CCMError(
"Building C* version {} failed. Attempted to delete {} but failed. This will need to be manually deleted"
.format(version, target_dir))
raise e
def compile_version(version, target_dir, verbose=False):
# compiling cassandra and the stress tool
logfile = os.path.join(__get_dir(), "last.log")
if verbose:
print_("Compiling Cassandra %s ..." % version)
with open(logfile, 'w') as lf:
lf.write("--- Cassandra Build -------------------\n")
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
while attempt < 3 and ret_val is not 0:
if attempt > 0:
lf.write("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt)
ret_val = subprocess.call([platform_binary('ant'),'jar'], cwd=target_dir, stdout=lf, stderr=lf)
attempt += 1
if ret_val is not 0:
raise CCMError("Error compiling Cassandra. See %s for details" % logfile)
except OSError as e:
raise CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile)
lf.write("\n\n--- cassandra/stress build ------------\n")
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
if subprocess.call([platform_binary('ant'), 'build'], cwd=stress_dir, stdout=lf, stderr=lf) is not 0:
if subprocess.call([platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=lf, stderr=lf) is not 0:
raise CCMError("Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise CCMError("Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)" % str(e))
def setup(version, verbose=False):
binary = True
fallback = True
if version.startswith('git:'):
clone_development(GIT_REPO, version, verbose=verbose)
return (version_directory(version), None)
elif version.startswith('local:'):
if LOCAL_GIT_REPO is None:
raise CCMError("LOCAL_GIT_REPO is not defined!")
clone_development(LOCAL_GIT_REPO, version, verbose=verbose)
return (version_directory(version), None)
elif version.startswith('binary:'):
version = version.replace('binary:', '')
fallback = False
elif version.startswith('github:'):
user_name, _ = github_username_and_branch_name(version)
clone_development(github_repo_for_user(user_name),
version,
verbose=verbose)
return (directory_name(version), None)
elif version.startswith('source:'):
version = version.replace('source:', '')
binary = False
fallback = False
if version in ('stable', 'oldstable', 'testing'):
version = get_tagged_version_numbers(version)[0]
cdir = version_directory(version)
if cdir is None:
try:
download_version(version, verbose=verbose, binary=binary)
cdir = version_directory(version)
except Exception as e:
# If we failed to download from ARCHIVE,
# then we build from source from the git repo,
# as it is more reliable.
# We don't do this if binary: or source: were
# explicitly specified.
if fallback:
common.warning(
"Downloading {} failed, due to {}. Trying to build from git instead."
.format(version, e))
version = 'git:cassandra-{}'.format(version)
clone_development(GIT_REPO, version, verbose=verbose)
return (version_directory(version), None)
else:
raise e
return (cdir, version)
def download_version(version, url=None, verbose=False, binary=False):
"""Download, extract, and build Cassandra tarball.
if binary == True, download precompiled tarball, otherwise build from source tarball.
"""
assert_jdk_valid_for_cassandra_version(version)
if binary:
u = "%s/%s/apache-cassandra-%s-bin.tar.gz" % (
ARCHIVE, version.split('-')[0], version) if url is None else url
else:
u = "%s/%s/apache-cassandra-%s-src.tar.gz" % (
ARCHIVE, version.split('-')[0], version) if url is None else url
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
try:
__download(u, target, show_progress=verbose)
if verbose:
print_("Extracting %s as version %s ..." % (target, version))
tar = tarfile.open(target)
dir = tar.next().name.split("/")[0]
tar.extractall(path=__get_dir())
tar.close()
target_dir = os.path.join(__get_dir(), version)
if os.path.exists(target_dir):
rmdirs(target_dir)
shutil.move(os.path.join(__get_dir(), dir), target_dir)
if binary:
# Binary installs don't have a build.xml that is needed
# for pulling the version from. Write the version number
# into a file to read later in common.get_version_from_build()
with open(os.path.join(target_dir, '0.version.txt'), 'w') as f:
f.write(version)
else:
compile_version(version, target_dir, verbose=verbose)
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: %s" %
str(e))
except CCMError as e:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
rmdirs(target_dir)
print_("Deleted %s due to error" % target_dir)
except:
raise CCMError(
"Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted"
% (version, target_dir))
raise e
def download_version(version, url=None, verbose=False, target_dir=None):
"""
Download, scylla relocatable package tarballs.
"""
try:
if os.path.exists(url) and url.endswith('.tar.gz'):
target = url
elif is_valid(url):
_, target = tempfile.mkstemp(suffix=".tar.gz", prefix="ccm-")
__download(url, target, show_progress=verbose)
else:
raise ArgumentError(
"unsupported url or file doesn't exist\n\turl={}".format(url))
if verbose:
print_("Extracting %s as version %s ..." % (target, version))
tar = tarfile.open(target)
tar.extractall(path=target_dir)
tar.close()
# add breadcrumb so we could list the origin of each part easily for debugging
# for example listing all the version we have in ccm scylla-repository
# find ~/.ccm/scylla-repository/*/ -iname source.txt | xargs cat
source_breadcrumb_file = os.path.join(target_dir, 'source.txt')
with open(source_breadcrumb_file, 'w') as f:
f.write("version=%s\n" % version)
f.write("url=%s\n" % url)
except urllib.error.URLError as e:
msg = "Invalid version %s" % version if url is None else "Invalid url %s" % url
msg = msg + " (underlying error is: %s)" % str(e)
raise ArgumentError(msg)
except tarfile.ReadError as e:
raise ArgumentError("Unable to uncompress downloaded file: %s" %
str(e))
except CCMError as e:
if target_dir:
# wipe out the directory if anything goes wrong.
try:
rmdirs(target_dir)
print_("Deleted %s due to error" % target_dir)
except:
raise CCMError(
"Downloading/extracting scylla version %s failed. Attempted to delete %s but failed. This will need to be manually deleted"
% (version, target_dir))
raise e
def __download(url, target, username=None, password=None, show_progress=False):
if username is not None:
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, password)
handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
opener = urllib.request.build_opener(handler)
urllib.request.install_opener(opener)
u = urllib.request.urlopen(url)
f = open(target, 'wb')
meta = u.info()
file_size = int(meta.get("Content-Length"))
if show_progress:
print_("Downloading %s to %s (%.3fMB)" %
(url, target, float(file_size) / (1024 * 1024)))
file_size_dl = 0
block_sz = 8192
status = None
attempts = 0
while file_size_dl < file_size:
buffer = u.read(block_sz)
if not buffer:
attempts = attempts + 1
if attempts >= 5:
raise CCMError(
"Error downloading file (nothing read after %i attempts, downloded only %i of %i bytes)"
% (attempts, file_size_dl, file_size))
time.sleep(0.5 * attempts)
continue
else:
attempts = 0
file_size_dl += len(buffer)
f.write(buffer)
if show_progress:
status = r"%10d [%3.2f%%]" % (file_size_dl,
file_size_dl * 100. / file_size)
status = chr(8) * (len(status) + 1) + status
print_(status, end='')
if show_progress:
print_("")
f.close()
u.close()
def __download(url, target, show_progress=False):
u = urllib.request.urlopen(url)
f = open(target, 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
if show_progress:
print_("Downloading %s to %s (%.3fMB)" %
(url, target, float(file_size) / (1024 * 1024)))
file_size_dl = 0
block_sz = 8192
status = None
attempts = 0
while file_size_dl < file_size:
buffer = u.read(block_sz)
if not buffer:
attempts = attempts + 1
if attempts >= 5:
raise CCMError(
"Error downloading file (nothing read after %i attempts, downloded only %i of %i bytes)"
% (attempts, file_size_dl, file_size))
time.sleep(0.5 * attempts)
continue
else:
attempts = 0
file_size_dl += len(buffer)
f.write(buffer)
if show_progress:
status = r"%10d [%3.2f%%]" % (file_size_dl,
file_size_dl * 100. / file_size)
status = chr(8) * (len(status) + 1) + status
print_(status, end='')
if show_progress:
print_("")
f.close()
u.close()
def clone_development(git_repo, version, verbose=False):
local_git_cache = os.path.join(__get_dir(), '_git_cache')
target_dir = os.path.join(__get_dir(), version.replace(
':', '_')) # handle git branches like 'git:trunk'.
git_branch = version[4:] # the part of the version after the 'git:'
logfile = os.path.join(__get_dir(), "last.log")
with open(logfile, 'w') as lf:
try:
#Checkout/fetch a local repository cache to reduce the number of
#remote fetches we need to perform:
if not os.path.exists(local_git_cache):
if verbose:
print_("Cloning Cassandra...")
out = subprocess.call(
['git', 'clone', '--mirror', git_repo, local_git_cache],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git clone"
else:
if verbose:
print_("Fetching Cassandra updates...")
out = subprocess.call(
['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'],
cwd=local_git_cache,
stdout=lf,
stderr=lf)
#Checkout the version we want from the local cache:
if not os.path.exists(target_dir):
# development branch doesn't exist. Check it out.
if verbose:
print_("Cloning Cassandra (from local cache)")
# git on cygwin appears to be adding `cwd` to the commands which is breaking clone
if sys.platform == "cygwin":
local_split = local_git_cache.split(os.sep)
target_split = target_dir.split(os.sep)
subprocess.call(
['git', 'clone', local_split[-1], target_split[-1]],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
else:
subprocess.call(
['git', 'clone', local_git_cache, target_dir],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
# now check out the right version
if verbose:
print_("Checking out requested branch (%s)" % git_branch)
out = subprocess.call(['git', 'checkout', git_branch],
cwd=target_dir,
stdout=lf,
stderr=lf)
if int(out) != 0:
raise CCMError(
"Could not check out git branch %s. Is this a valid branch name? (see last.log for details)"
% git_branch)
# now compile
compile_version(git_branch, target_dir, verbose)
else: # branch is already checked out. See if it is behind and recompile if needed.
out = subprocess.call(['git', 'fetch', 'origin'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git fetch"
status = subprocess.Popen(['git', 'status', '-sb'],
cwd=target_dir,
stdout=subprocess.PIPE,
stderr=lf).communicate()[0]
if str(status).find('[behind') > -1:
if verbose:
print_("Branch is behind, recompiling")
out = subprocess.call(['git', 'pull'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git pull"
out = subprocess.call(
[platform_binary('ant'), 'realclean'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not run 'ant realclean'"
# now compile
compile_version(git_branch, target_dir, verbose)
except:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
shutil.rmtree(target_dir)
print_("Deleted %s due to error" % target_dir)
except:
raise CCMError(
"Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted"
% (version, target_dir))
raise
def setup(version, verbose=False, elassandra_version=None):
binary = True
fallback = True
if version.startswith('git:'):
clone_development(GIT_REPO, version, verbose=verbose, elassandra_version=elassandra_version)
return (elassandra_version_directory(version, elassandra_version), None)
elif version.startswith('local:'):
# local: slugs take the form of: "local:/some/path/:somebranch"
try:
_, path, branch = version.split(':')
except ValueError:
raise CCMError("local version ({}) appears to be invalid. Please format as local:/some/path/:somebranch".format(version))
clone_development(path, version, verbose=verbose, elassandra_version=elassandra_version)
version_dir = elassandra_version_directory(version, elassandra_version)
if version_dir is None:
raise CCMError("Path provided in local slug appears invalid ({})".format(path))
return (version_dir, None)
elif version.startswith('binary:'):
version = version.replace('binary:', '')
fallback = False
elif version.startswith('github:'):
user_name, _ = github_username_and_branch_name(version)
clone_development(github_repo_for_user(user_name), version, verbose=verbose, elassandra_version=elassandra_version)
return (elassandra_version_directory(version, elassandra_version), None)
elif version.startswith('source:'):
version = version.replace('source:', '')
binary = False
fallback = False
elif version.startswith('alias:'):
alias = version.split(":")[1].split("/")[0]
try:
git_repo = CCM_CONFIG.get("aliases", alias)
clone_development(git_repo, version, verbose=verbose, alias=True, elassandra_version=elassandra_version)
return (elassandra_version_directory(version, elassandra_version), None)
except ConfigParser.NoOptionError as e:
common.warning("Unable to find alias {} in configuration file.".format(alias))
raise e
if version in ('stable', 'oldstable', 'testing'):
version = get_tagged_version_numbers(version)[0]
if elassandra_version is None:
elassandra_version = version
cdir = version_directory(elassandra_version)
if cdir is None:
try:
download_version(elassandra_version, verbose=verbose, binary=binary)
cdir = version_directory(elassandra_version)
except Exception as e:
# If we failed to download from ARCHIVE,
# then we build from source from the git repo,
# as it is more reliable.
# We don't do this if binary: or source: were
# explicitly specified.
if fallback and False: # skip fallback, would not work with elassandra that way
common.warning("Downloading {} failed, due to {}. Trying to build from git instead.".format(version, e))
version = 'git:cassandra-{}'.format(version)
clone_development(GIT_REPO, version, verbose=verbose, elassandra_version=elassandra_version)
return (version_directory(version), None)
else:
raise e
return (cdir, elassandra_version)
def clone_development(git_repo, version, verbose=False, alias=False, elassandra_version=None):
print_(git_repo, version)
target_dir = directory_name(version)
assert target_dir
if 'github' in version:
git_repo_name, git_branch = github_username_and_branch_name(version)
elif 'local:' in version:
git_repo_name = 'local_{}'.format(git_repo) # add git repo location to distinguish cache location for differing repos
git_branch = version.split(':')[-1] # last token on 'local:...' slugs should always be branch name
elif alias:
git_repo_name = 'alias_{}'.format(version.split('/')[0].split(':')[-1])
git_branch = version.split('/')[-1]
else:
git_repo_name = 'strapdata'
git_branch = version.split(':', 1)[1]
local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name)
logfile = lastlogfilename()
logger = get_logger(logfile)
try:
# Checkout/fetch a local repository cache to reduce the number of
# remote fetches we need to perform:
if not os.path.exists(local_git_cache):
common.info("Cloning Elassandra...")
process = subprocess.Popen(
['git', 'clone', '--mirror', git_repo, local_git_cache],
cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
assert out == 0, "Could not do a git clone"
else:
common.info("Fetching Elassandra updates...")
process = subprocess.Popen(
['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'],
cwd=local_git_cache, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
assert out == 0, "Could not update git"
# Checkout the version we want from the local cache:
if not os.path.exists(target_dir):
# development branch doesn't exist. Check it out.
common.info("Cloning Elassandra (from local cache)")
# git on cygwin appears to be adding `cwd` to the commands which is breaking clone
if sys.platform == "cygwin":
local_split = local_git_cache.split(os.sep)
target_split = target_dir.split(os.sep)
process = subprocess.Popen(
['git', 'clone', local_split[-1], target_split[-1]],
cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
assert out == 0, "Could not do a git clone"
else:
process = subprocess.Popen(
['git', 'clone', local_git_cache, target_dir],
cwd=__get_dir(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
assert out == 0, "Could not do a git clone"
# determine if the request is for a branch
is_branch = False
try:
branch_listing = subprocess.check_output(['git', 'branch', '--all'], cwd=target_dir).decode('utf-8')
branches = [b.strip() for b in branch_listing.replace('remotes/origin/', '').split()]
is_branch = git_branch in branches
except subprocess.CalledProcessError as cpe:
common.error("Error Running Branch Filter: {}\nAssumming request is not for a branch".format(cpe.output))
# now check out the right version
branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag'
common.info("Checking out requested {} ({})".format(branch_or_sha_tag, git_branch))
if is_branch:
# we use checkout -B with --track so we can specify that we want to track a specific branch
# otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360
# we use -B instead of -b so we reset branches that already exist and create a new one otherwise
process = subprocess.Popen(['git', 'checkout', '-B', git_branch,
'--track', 'origin/{git_branch}'.format(git_branch=git_branch)],
cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
else:
process = subprocess.Popen(
['git', 'checkout', git_branch],
cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
if int(out) != 0:
raise CCMError('Could not check out git branch {branch}. '
'Is this a valid branch name? (see {lastlog} or run '
'"ccm showlastlog" for details)'.format(
branch=git_branch, lastlog=logfile
))
# now compile
compile_version(git_branch, target_dir, verbose, elassandra_version=elassandra_version)
else: # branch is already checked out. See if it is behind and recompile if needed.
process = subprocess.Popen(
['git', 'fetch', 'origin'],
cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
assert out == 0, "Could not do a git fetch"
process = subprocess.Popen(['git', 'status', '-sb'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
_, status, _ = log_info(process, logger)
if str(status).find('[behind') > -1: # If `status` looks like '## cassandra-2.2...origin/cassandra-2.2 [behind 9]\n'
common.info("Branch is behind, recompiling")
process = subprocess.Popen(['git', 'pull'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
assert out == 0, "Could not do a git pull"
process = subprocess.Popen([platform_binary('ant'), 'realclean'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _, _ = log_info(process, logger)
assert out == 0, "Could not run 'ant realclean'"
# now compile
compile_version(git_branch, target_dir, verbose, elassandra_version=elassandra_version)
elif re.search('\[.*?(ahead|behind).*?\]', status.decode("utf-8")) is not None: # status looks like '## trunk...origin/trunk [ahead 1, behind 29]\n'
# If we have diverged in a way that fast-forward merging cannot solve, raise an exception so the cache is wiped
common.error("Could not ascertain branch status, please resolve manually.")
raise Exception
else: # status looks like '## cassandra-2.2...origin/cassandra-2.2\n'
common.debug("Branch up to date, not pulling.")
except Exception as e:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
rmdirs(target_dir)
common.error("Deleted {} due to error".format(target_dir))
except:
print_('Building C* version {version} failed. Attempted to delete {target_dir}'
'but failed. This will need to be manually deleted'.format(
version=version,
target_dir=target_dir
))
finally:
raise e
def compile_version(version, target_dir, verbose=False):
# compiling cassandra and the stress tool
logfile = lastlogfilename()
logger = get_logger(logfile)
common.info("Compiling Cassandra {} ...".format(version))
logger.info("--- Cassandra Build -------------------\n")
env = update_java_version(
install_dir=target_dir,
for_build=True,
info_message='Cassandra {} build'.format(version))
default_build_properties = os.path.join(common.get_default_path(),
'build.properties.default')
if os.path.exists(default_build_properties):
target_build_properties = os.path.join(target_dir, 'build.properties')
logger.info("Copying %s to %s\n" %
(default_build_properties, target_build_properties))
shutil.copyfile(default_build_properties, target_build_properties)
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
gradlew = os.path.join(target_dir, platform_binary('gradlew'))
if os.path.exists(gradlew):
cmd = [gradlew, 'jar']
else:
# No gradle, use ant
cmd = [platform_binary('ant'), 'jar']
if get_jdk_version_int() >= 11:
cmd.append('-Duse.jdk11=true')
while attempt < 3 and ret_val != 0:
if attempt > 0:
logger.info("\n\n`{}` failed. Retry #{}...\n\n".format(
' '.join(cmd), attempt))
process = subprocess.Popen(cmd,
cwd=target_dir,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret_val, stdout, stderr = log_info(process, logger)
attempt += 1
if ret_val != 0:
raise CCMError(
'Error compiling Cassandra. See {logfile} or run '
'"ccm showlastlog" for details, stdout=\'{stdout}\' stderr=\'{stderr}\''
.format(logfile=logfile,
stdout=stdout.decode(),
stderr=stderr.decode()))
except OSError as e:
raise CCMError(
"Error compiling Cassandra. Is ant installed? See %s for details" %
logfile)
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(
build_xml): # building stress separately is only necessary pre-1.1
logger.info("\n\n--- cassandra/stress build ------------\n")
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(
full_path,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP
| stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
process = subprocess.Popen([platform_binary('ant'), 'build'],
cwd=stress_dir,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val != 0:
process = subprocess.Popen(
[platform_binary('ant'), 'stress-build'],
cwd=target_dir,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val != 0:
raise CCMError(
"Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise CCMError(
"Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)"
% str(e))
def setup(version, verbose=False):
binary = True
fallback = True
if version.startswith('git:'):
clone_development(GIT_REPO, version, verbose=verbose)
return (version_directory(version), None)
elif version.startswith('local:'):
# local: slugs take the form of: "local:/some/path/:somebranch"
try:
_, path, branch = version.split(':')
except ValueError:
raise CCMError(
"local version ({}) appears to be invalid. Please format as local:/some/path/:somebranch"
.format(version))
clone_development(path, version, verbose=verbose)
version_dir = version_directory(version)
if version_dir is None:
raise CCMError(
"Path provided in local slug appears invalid ({})".format(
path))
return (version_dir, None)
elif version.startswith('binary:'):
version = version.replace('binary:', '')
fallback = False
elif version.startswith('github:'):
user_name, _ = github_username_and_branch_name(version)
# make sure to use http for cloning read-only repos such as 'github:apache/cassandra-2.1'
if user_name == "apache":
clone_development(GITHUB_REPO, version, verbose=verbose)
else:
clone_development(github_repo_for_user(user_name),
version,
verbose=verbose)
return (directory_name(version), None)
elif version.startswith('source:'):
version = version.replace('source:', '')
elif version.startswith('clone:'):
# locally present C* source tree
version = version.replace('clone:', '')
return (version, None)
elif version.startswith('alias:'):
alias = version.split(":")[1].split("/")[0]
try:
git_repo = CCM_CONFIG.get("aliases", alias)
clone_development(git_repo, version, verbose=verbose, alias=True)
return (directory_name(version), None)
except ConfigParser.NoOptionError as e:
common.warning(
"Unable to find alias {} in configuration file.".format(alias))
raise e
if version in ('stable', 'oldstable', 'testing'):
version = get_tagged_version_numbers(version)[0]
cdir = version_directory(version)
if cdir is None:
try:
download_version(version, verbose=verbose, binary=binary)
cdir = version_directory(version)
except Exception as e:
# If we failed to download from ARCHIVE,
# then we build from source from the git repo,
# as it is more reliable.
# We don't do this if binary: or source: were
# explicitly specified.
if fallback:
common.warning(
"Downloading {} failed, trying to build from git instead.\n"
"The error was: {}".format(version, e))
version = 'git:cassandra-{}'.format(version)
clone_development(GIT_REPO, version, verbose=verbose)
return (version_directory(version), None)
else:
raise e
return (cdir, version)
def clone_development(git_repo, version, verbose=False):
print_(git_repo, version)
target_dir = directory_name(version)
assert target_dir
if 'github' in version:
git_repo_name, git_branch = github_username_and_branch_name(version)
else:
git_repo_name = 'apache'
git_branch = version.split(':', 1)[1]
local_git_cache = os.path.join(__get_dir(), '_git_cache_' + git_repo_name)
logfile = lastlogfilename()
with open(logfile, 'w') as lf:
try:
# Checkout/fetch a local repository cache to reduce the number of
# remote fetches we need to perform:
if not os.path.exists(local_git_cache):
if verbose:
print_("Cloning Cassandra...")
out = subprocess.call(
['git', 'clone', '--mirror', git_repo, local_git_cache],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git clone"
else:
if verbose:
print_("Fetching Cassandra updates...")
out = subprocess.call(
['git', 'fetch', '-fup', 'origin', '+refs/*:refs/*'],
cwd=local_git_cache,
stdout=lf,
stderr=lf)
# Checkout the version we want from the local cache:
if not os.path.exists(target_dir):
# development branch doesn't exist. Check it out.
if verbose:
print_("Cloning Cassandra (from local cache)")
# git on cygwin appears to be adding `cwd` to the commands which is breaking clone
if sys.platform == "cygwin":
local_split = local_git_cache.split(os.sep)
target_split = target_dir.split(os.sep)
subprocess.call(
['git', 'clone', local_split[-1], target_split[-1]],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
else:
subprocess.call(
['git', 'clone', local_git_cache, target_dir],
cwd=__get_dir(),
stdout=lf,
stderr=lf)
# determine if the request is for a branch
is_branch = False
try:
branch_listing = subprocess.check_output(
['git', 'branch', '--all'],
cwd=target_dir).decode('utf-8')
branches = [
b.strip() for b in branch_listing.replace(
'remotes/origin/', '').split()
]
is_branch = git_branch in branches
except subprocess.CalledProcessError as cpe:
print_(
"Error Running Branch Filter: {}\nAssumming request is not for a branch"
.format(cpe.output))
# now check out the right version
if verbose:
branch_or_sha_tag = 'branch' if is_branch else 'SHA/tag'
print_("Checking out requested {} ({})".format(
branch_or_sha_tag, git_branch))
if is_branch:
# we use checkout -B with --track so we can specify that we want to track a specific branch
# otherwise, you get errors on branch names that are also valid SHAs or SHA shortcuts, like 10360
# we use -B instead of -b so we reset branches that already exist and create a new one otherwise
out = subprocess.call([
'git', 'checkout', '-B', git_branch, '--track',
'origin/{git_branch}'.format(git_branch=git_branch)
],
cwd=target_dir,
stdout=lf,
stderr=lf)
else:
out = subprocess.call(['git', 'checkout', git_branch],
cwd=target_dir,
stdout=lf,
stderr=lf)
if int(out) != 0:
raise CCMError(
'Could not check out git branch {branch}. '
'Is this a valid branch name? (see {lastlog} or run '
'"ccm showlastlog" for details)'.format(
branch=git_branch, lastlog=logfile))
# now compile
compile_version(git_branch, target_dir, verbose)
else: # branch is already checked out. See if it is behind and recompile if needed.
out = subprocess.call(['git', 'fetch', 'origin'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git fetch"
status = subprocess.Popen(['git', 'status', '-sb'],
cwd=target_dir,
stdout=subprocess.PIPE,
stderr=lf).communicate()[0]
if str(status).find('[behind') > -1:
if verbose:
print_("Branch is behind, recompiling")
out = subprocess.call(['git', 'pull'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not do a git pull"
out = subprocess.call(
[platform_binary('ant'), 'realclean'],
cwd=target_dir,
stdout=lf,
stderr=lf)
assert out == 0, "Could not run 'ant realclean'"
# now compile
compile_version(git_branch, target_dir, verbose)
except:
# wipe out the directory if anything goes wrong. Otherwise we will assume it has been compiled the next time it runs.
try:
rmdirs(target_dir)
print_("Deleted %s due to error" % target_dir)
except:
raise CCMError(
"Building C* version %s failed. Attempted to delete %s but failed. This will need to be manually deleted"
% (version, target_dir))
raise