def validate(self, parser, options, args, cluster_name=False, node_name=False, load_cluster=False, load_node=True):
self.options = options
self.args = args
if options.config_dir is None:
self.path = common.get_default_path()
else:
self.path = options.config_dir
if cluster_name:
if len(args) == 0:
print_('Missing cluster name', file=sys.stderr)
parser.print_help()
exit(1)
self.name = args[0]
if node_name:
if len(args) == 0:
print_('Missing node name', file=sys.stderr)
parser.print_help()
exit(1)
self.name = args[0]
if load_cluster:
self.cluster = self._load_current_cluster()
if node_name and load_node:
try:
self.node = self.cluster.nodes[self.name]
except KeyError:
print_('Unknown node %s in cluster %s' % (self.name, self.cluster.name), file=sys.stderr)
exit(1)
def validate(self, parser, options, args, cluster_name=False, node_name=False, load_cluster=False, load_node=True):
self.options = options
self.args = args
if options.config_dir is None:
self.path = common.get_default_path()
else:
self.path = options.config_dir
if cluster_name:
if len(args) == 0:
print_('Missing cluster name', file=sys.stderr)
parser.print_help()
exit(1)
self.name = args[0]
if node_name:
if len(args) == 0:
print_('Missing node name', file=sys.stderr)
parser.print_help()
exit(1)
self.name = args[0]
if load_cluster:
# FIXME: here must take info from cluster.cfg
# TODO: from scylla_docker_cluster.py we need to save all data into cluster.cfg (and also node.cfg)
self.cluster = self._load_current_cluster()
if node_name and load_node:
try:
self.node = self.cluster.nodes[self.name]
except KeyError:
print_('Unknown node %s in cluster %s' % (self.name, self.cluster.name), file=sys.stderr)
exit(1)
def compile_version(version, target_dir, verbose=False):
assert_jdk_valid_for_cassandra_version(get_version_from_build(target_dir))
# compiling cassandra and the stress tool
logfile = lastlogfilename()
logger = get_logger(logfile)
common.info("Compiling Cassandra {} ...".format(version))
logger.info("--- Cassandra Build -------------------\n")
default_build_properties = os.path.join(common.get_default_path(), 'build.properties.default')
if os.path.exists(default_build_properties):
target_build_properties = os.path.join(target_dir, 'build.properties')
logger.info("Copying %s to %s\n" % (default_build_properties, target_build_properties))
shutil.copyfile(default_build_properties, target_build_properties)
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
while attempt < 3 and ret_val is not 0:
if attempt > 0:
logger.info("\n\n`ant jar` failed. Retry #%s...\n\n" % attempt)
process = subprocess.Popen([platform_binary('ant'), 'jar'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
attempt += 1
if ret_val is not 0:
raise CCMError('Error compiling Cassandra. See {logfile} or run '
'"ccm showlastlog" for details'.format(logfile=logfile))
except OSError as e:
raise CCMError("Error compiling Cassandra. Is ant installed? See %s for details" % logfile)
logger.info("\n\n--- cassandra/stress build ------------\n")
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(build_xml): # building stress separately is only necessary pre-1.1
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(full_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
process = subprocess.Popen([platform_binary('ant'), 'build'], cwd=stress_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val is not 0:
process = subprocess.Popen([platform_binary('ant'), 'stress-build'], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val is not 0:
raise CCMError("Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise CCMError("Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)" % str(e))
def cluster_create_start_test(self):
args = ['-n', '1', '-s']
self.validate_output(self.create_cmd(args))
pidfile = os.path.join(common.get_default_path(), 'test', 'node1', 'cassandra.pid')
with open(pidfile, 'r') as f:
pid = int(f.readline().strip())
os.kill(pid, 0)
def cluster_create_vnodes_test(self):
args = ['-n', '1', '--vnodes']
self.validate_output(self.create_cmd(args))
yaml_path = os.path.join(common.get_default_path(), 'test', 'node1', 'conf', 'cassandra.yaml')
with open(yaml_path, 'r') as f:
data = yaml.load(f)
self.assertEqual(256, data['num_tokens'])
def cluster_create_vnodes_test(self):
args = ['-n', '1', '--vnodes']
self.validate_output(self.create_cmd(args))
yaml_path = os.path.join(common.get_default_path(), 'test', 'node1', 'conf', 'cassandra.yaml')
with open(yaml_path, 'r') as f:
data = yaml.safe_load(f)
self.assertEqual(256, data['num_tokens'])
def cluster_create_start_test(self):
args = ['-n', '1', '-s']
self.validate_output(self.create_cmd(args))
pidfile = os.path.join(common.get_default_path(), 'test', 'node1', 'cassandra.pid')
with open(pidfile, 'r') as f:
pid = int(f.readline().strip())
os.kill(pid, 0)
def cluster_create_no_switch_test(self):
self.create_cmd(args=None, name='not_test')
args = ['--no-switch']
self.validate_output(self.create_cmd(args))
self.assertEqual('not_test', common.current_cluster_name(common.get_default_path()))
p = subprocess.Popen(['ccm', 'remove'])
p.wait()
p = subprocess.Popen(['ccm', 'switch', 'test'])
p.wait()
p = subprocess.Popen(['ccm', 'remove'])
p.wait()
def cluster_create_no_switch_test(self):
self.create_cmd(args=None, name='not_test')
args = ['--no-switch']
self.validate_output(self.create_cmd(args))
self.assertEqual('not_test', common.current_cluster_name(common.get_default_path()))
p = subprocess.Popen(['ccm', 'remove'])
p.wait()
p = subprocess.Popen(['ccm', 'switch', 'test'])
p.wait()
p = subprocess.Popen(['ccm', 'remove'])
p.wait()
def load_credentials_from_file(self, dse_credentials_file):
# Use .dse.ini if it exists in the default .ccm directory.
if dse_credentials_file is None:
creds_file = os.path.join(common.get_default_path(), '.dse.ini')
if os.path.isfile(creds_file):
dse_credentials_file = creds_file
if dse_credentials_file is not None:
parser = ConfigParser.ConfigParser()
parser.read(dse_credentials_file)
if parser.has_section('dse_credentials'):
if parser.has_option('dse_credentials', 'dse_username'):
self.dse_username = parser.get('dse_credentials', 'dse_username')
if parser.has_option('dse_credentials', 'dse_password'):
self.dse_password = parser.get('dse_credentials', 'dse_password')
else:
common.warning("{} does not contain a 'dse_credentials' section.".format(dse_credentials_file))
def load_credentials_from_file(self, dse_credentials_file):
# Use .dse.ini if it exists in the default .ccm directory.
if dse_credentials_file is None:
creds_file = os.path.join(common.get_default_path(), '.dse.ini')
if os.path.isfile(creds_file):
dse_credentials_file = creds_file
if dse_credentials_file is not None:
parser = ConfigParser.RawConfigParser()
parser.read(dse_credentials_file)
if parser.has_section('dse_credentials'):
if parser.has_option('dse_credentials', 'dse_username'):
self.dse_username = parser.get('dse_credentials', 'dse_username')
if parser.has_option('dse_credentials', 'dse_password'):
self.dse_password = parser.get('dse_credentials', 'dse_password')
else:
common.warning("{} does not contain a 'dse_credentials' section.".format(dse_credentials_file))
def load_credentials_from_file(self, dse_credentials_file):
# Use .dse.ini if it exists in the default .ccm directory.
if dse_credentials_file is None:
creds_file = os.path.join(common.get_default_path(), ".dse.ini")
if os.path.isfile(creds_file):
dse_credentials_file = creds_file
if dse_credentials_file is not None:
parser = ConfigParser.ConfigParser()
parser.read(dse_credentials_file)
if parser.has_section("dse_credentials"):
if parser.has_option("dse_credentials", "dse_username"):
self.dse_username = parser.get("dse_credentials", "dse_username")
if parser.has_option("dse_credentials", "dse_password"):
self.dse_password = parser.get("dse_credentials", "dse_password")
else:
print_(
"Warning: {} does not contain a 'dse_credentials' section.".format(dse_credentials_file),
file=sys.stderr,
)
def __init__(self,
test_id,
use_scylla=True,
relocatable_version=None,
docker_image=None,
scylla_manager_package=None):
if scylla_manager_package:
os.environ['SCYLLA_MANAGER_PACKAGE'] = ''
else:
try:
del os.environ['SCYLLA_MANAGER_PACKAGE']
except Exception:
pass
self.name = f"{self.__class__.__name__}-{test_id}"
self.ccm_bin = os.path.join(os.curdir, "ccm")
self.cluster_dir = os.path.join(common.get_default_path(), self.name)
self.use_scylla = use_scylla
self.relocatable_version = relocatable_version
self.docker_image = docker_image
self._process = None
def _tuple_version(version_string):
if '-' in version_string:
version_string = version_string[:version_string.index('-')]
return tuple([int(p) for p in version_string.split('.')])
USE_CASS_EXTERNAL = bool(os.getenv('USE_CASS_EXTERNAL', False))
default_cassandra_version = '2.1.3'
if USE_CASS_EXTERNAL:
if CCMClusterFactory:
# see if the external instance is running in ccm
path = common.get_default_path()
name = common.current_cluster_name(path)
CCM_CLUSTER = CCMClusterFactory.load(common.get_default_path(), name)
CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True)
# Not sure what's going on, but the server version query
# hangs in python3. This appears to be related to running inside of
# nosetests, and only for this query that would run while loading the
# module.
# This is a hack to make it run with default cassandra version for PY3.
# Not happy with it, but need to move on for now.
if not six.PY3:
cass_ver, _ = get_server_versions()
default_cassandra_version = '.'.join('%d' % i for i in cass_ver)
else:
if not os.getenv('CASSANDRA_VERSION'):
def __get_dir():
repo = os.path.join(get_default_path(), 'scylla-repository')
if not os.path.exists(repo):
os.mkdir(repo)
return repo
def __get_dir():
repo = os.path.join(get_default_path(), "repository")
if not os.path.exists(repo):
os.mkdir(repo)
return repo
def compile_version(version, target_dir, verbose=False):
# compiling cassandra and the stress tool
logfile = lastlogfilename()
logger = get_logger(logfile)
common.info("Compiling Cassandra {} ...".format(version))
logger.info("--- Cassandra Build -------------------\n")
env = update_java_version(
install_dir=target_dir,
for_build=True,
info_message='Cassandra {} build'.format(version))
default_build_properties = os.path.join(common.get_default_path(),
'build.properties.default')
if os.path.exists(default_build_properties):
target_build_properties = os.path.join(target_dir, 'build.properties')
logger.info("Copying %s to %s\n" %
(default_build_properties, target_build_properties))
shutil.copyfile(default_build_properties, target_build_properties)
try:
# Patch for pending Cassandra issue: https://issues.apache.org/jira/browse/CASSANDRA-5543
# Similar patch seen with buildbot
attempt = 0
ret_val = 1
gradlew = os.path.join(target_dir, platform_binary('gradlew'))
if os.path.exists(gradlew):
cmd = [gradlew, 'jar']
else:
# No gradle, use ant
cmd = [platform_binary('ant'), 'jar']
if get_jdk_version_int() >= 11:
cmd.append('-Duse.jdk11=true')
while attempt < 3 and ret_val != 0:
if attempt > 0:
logger.info("\n\n`{}` failed. Retry #{}...\n\n".format(
' '.join(cmd), attempt))
process = subprocess.Popen(cmd,
cwd=target_dir,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret_val, stdout, stderr = log_info(process, logger)
attempt += 1
if ret_val != 0:
raise CCMError(
'Error compiling Cassandra. See {logfile} or run '
'"ccm showlastlog" for details, stdout=\'{stdout}\' stderr=\'{stderr}\''
.format(logfile=logfile,
stdout=stdout.decode(),
stderr=stderr.decode()))
except OSError as e:
raise CCMError(
"Error compiling Cassandra. Is ant installed? See %s for details" %
logfile)
stress_dir = os.path.join(target_dir, "tools", "stress") if (
version >= "0.8.0") else \
os.path.join(target_dir, "contrib", "stress")
build_xml = os.path.join(stress_dir, 'build.xml')
if os.path.exists(
build_xml): # building stress separately is only necessary pre-1.1
logger.info("\n\n--- cassandra/stress build ------------\n")
try:
# set permissions correctly, seems to not always be the case
stress_bin_dir = os.path.join(stress_dir, 'bin')
for f in os.listdir(stress_bin_dir):
full_path = os.path.join(stress_bin_dir, f)
os.chmod(
full_path,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP
| stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
process = subprocess.Popen([platform_binary('ant'), 'build'],
cwd=stress_dir,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val != 0:
process = subprocess.Popen(
[platform_binary('ant'), 'stress-build'],
cwd=target_dir,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret_val, _, _ = log_info(process, logger)
if ret_val != 0:
raise CCMError(
"Error compiling Cassandra stress tool. "
"See %s for details (you will still be able to use ccm "
"but not the stress related commands)" % logfile)
except IOError as e:
raise CCMError(
"Error compiling Cassandra stress tool: %s (you will "
"still be able to use ccm but not the stress related commands)"
% str(e))
def _tuple_version(version_string):
if '-' in version_string:
version_string = version_string[:version_string.index('-')]
return tuple([int(p) for p in version_string.split('.')])
USE_CASS_EXTERNAL = bool(os.getenv('USE_CASS_EXTERNAL', False))
default_cassandra_version = '2.1.3'
if USE_CASS_EXTERNAL:
if CCMClusterFactory:
# see if the external instance is running in ccm
path = common.get_default_path()
name = common.current_cluster_name(path)
CCM_CLUSTER = CCMClusterFactory.load(common.get_default_path(), name)
CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True)
# Not sure what's going on, but the server version query
# hangs in python3. This appears to be related to running inside of
# nosetests, and only for this query that would run while loading the
# module.
# This is a hack to make it run with default cassandra version for PY3.
# Not happy with it, but need to move on for now.
if not six.PY3:
cass_ver, _ = get_server_versions()
default_cassandra_version = '.'.join('%d' % i for i in cass_ver)
else:
if not os.getenv('CASSANDRA_VERSION'):
def cluster_create_cassandra_dir_test(self):
c_dir = common.get_default_path()
c_dir = os.path.join(c_dir, 'repository')
c_dir = os.path.join(c_dir, os.listdir(c_dir)[0])
args = ['--install-dir', c_dir]
self.validate_output(self.create_cmd(args, version=None))
def cluster_create_cassandra_dir_test(self):
c_dir = common.get_default_path()
c_dir = os.path.join(c_dir, 'repository')
c_dir = os.path.join(c_dir, os.listdir(c_dir)[0])
args = ['--install-dir', c_dir]
self.validate_output(self.create_cmd(args, version=None))