def run_json2sstable(self, in_file, ks, cf, keyspace=None, datafile=None, column_families=None, enumerate_keys=False): cdir = self.get_cassandra_dir() if self.cluster.version() >= "2.1": json2sstable = common.join_bin(cdir, os.path.join('tools', 'bin'), 'json2sstable') else: json2sstable = common.join_bin(cdir, 'bin', 'json2sstable') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile,keyspace,column_families) for datafile in datafiles: in_file_name = os.path.abspath(in_file.name) args = shlex.split("{json2sstable} -s -K {ks} -c {cf} {in_file_name} {datafile}".format(**locals())) subprocess.call(args, env=env)
def dsetool(self, cmd): env = common.make_dse_env(self.get_install_dir(), self.get_path()) dsetool = common.join_bin(self.get_install_dir(), 'bin', 'dsetool') args = [dsetool, '-h', 'localhost', '-j', str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def run_sstable2json(self, out_file=sys.__stdout__, keyspace=None, datafile=None, column_families=None, enumerate_keys=False): cdir = self.get_cassandra_dir() if self.cluster.version() >= "2.1": sstable2json = common.join_bin(cdir, os.path.join('tools', 'bin'), 'sstable2json') else: sstable2json = common.join_bin(cdir, 'bin', 'sstable2json') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile,keyspace,column_families) for datafile in datafiles: print_("-- {0} -----".format(os.path.basename(datafile))) args = [ sstable2json , datafile ] if enumerate_keys: args = args + ["-e"] subprocess.call(args, env=env, stdout=out_file) print_("")
def nodetool(self, cmd, username=None, password=None, capture_output=True, wait=True): """ Setting wait=False makes it impossible to detect errors, if capture_output is also False. wait=False allows us to return while nodetool is still running. """ if capture_output and not wait: raise common.ArgumentError("Cannot set capture_output while wait is False.") env = self.get_env() nodetool = common.join_bin(self.get_install_dir(), 'bin', 'nodetool') args = [nodetool, '-h', 'localhost', '-p', str(self.jmx_port)] if username is not None: args += [ '-u', username] if password is not None: args += [ '-pw', password] args += cmd.split() if capture_output: p = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() else: p = subprocess.Popen(args, env=env) stdout, stderr = None, None if wait: exit_status = p.wait() if exit_status != 0: raise NodetoolError(" ".join(args), exit_status, stdout, stderr) return stdout, stderr
def spark(self, spark_options=[]): env = common.make_dse_env(self.get_install_dir(), self.get_path()) dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'spark'] args += spark_options p = subprocess.Popen(args, env=env) p.wait()
def dsetool(self, cmd): env = self.get_env() dsetool = common.join_bin(self.get_install_dir(), 'bin', 'dsetool') args = [dsetool, '-h', 'localhost', '-j', str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cqlsh') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] if self.cluster.version() >= "2.1": port = self.network_interfaces['binary'][1] else: port = self.network_interfaces['thrift'][1] args = cqlsh_options + [ host, str(port) ] sys.stdout.flush() if cmds is None: os.execve(cli, [ common.platform_binary('cqlsh') ] + args, env) else: p = subprocess.Popen([ cli ] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) for cmd in cmds.split(';'): p.stdin.write(cmd + ';\n') p.stdin.write("quit;\n") p.wait() for err in p.stderr: print_("(EE) ", err, end='') if show_output: i = 0 for log in p.stdout: # first four lines are not interesting if i >= 4: print_(log, end='') i = i + 1
def run_cli(self, cmds=None, show_output=False, cli_options=[]): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cassandra-cli') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] port = self.network_interfaces['thrift'][1] args = [ '-h', host, '-p', str(port) , '--jmxport', str(self.jmx_port) ] + cli_options sys.stdout.flush() if cmds is None: os.execve(cli, [ common.platform_binary('cassandra-cli') ] + args, env) else: p = subprocess.Popen([ cli ] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) for cmd in cmds.split(';'): p.stdin.write(cmd + ';\n') p.stdin.write("quit;\n") p.wait() for err in p.stderr: print_("(EE) ", err, end='') if show_output: i = 0 for log in p.stdout: # first four lines are not interesting if i >= 4: print_(log, end='') i = i + 1
def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, "bin", "cqlsh") env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces["thrift"][0] if self.cluster.version() >= "2.1": port = self.network_interfaces["binary"][1] else: port = self.network_interfaces["thrift"][1] args = cqlsh_options + [host, str(port)] sys.stdout.flush() if cmds is None: os.execve(cli, [common.platform_binary("cqlsh")] + args, env) else: p = subprocess.Popen( [cli] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE ) for cmd in cmds.split(";"): p.stdin.write(cmd + ";\n") p.stdin.write("quit;\n") p.wait() for err in p.stderr: print_("(EE) ", err, end="") if show_output: i = 0 for log in p.stdout: # first four lines are not interesting if i >= 4: print_(log, end="") i = i + 1
def dsetool(self, cmd): env = self.get_env() extension.append_to_client_env(self, env) dsetool = common.join_bin(self.get_install_dir(), 'bin', 'dsetool') args = [dsetool, '-h', 'localhost', '-j', str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def cli(self): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cassandra-cli') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] port = self.network_interfaces['thrift'][1] args = [ '-h', host, '-p', str(port) , '--jmxport', str(self.jmx_port) ] return CliSession(subprocess.Popen([ cli ] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE))
def sqoop(self, sqoop_options=[]): env = common.make_dse_env(self.get_install_dir(), self.get_path()) env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'sqoop'] args += sqoop_options p = subprocess.Popen(args, env=env) p.wait()
def dsetool(self, cmd): env = common.make_dse_env(self.get_install_dir(), self.get_path()) host = self.address() dsetool = common.join_bin(self.get_install_dir(), "bin", "dsetool") args = [dsetool, "-h", host, "-j", str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def dsetool(self, cmd): env = common.make_dse_env(self.get_install_dir(), self.get_path()) host = self.address() dsetool = common.join_bin(self.get_install_dir(), 'bin', 'dsetool') args = [dsetool, '-h', host, '-j', str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def spark(self, spark_options=[]): env = common.make_dse_env(self.get_install_dir(), self.get_path()) env["JMX_PORT"] = self.jmx_port dse = common.join_bin(self.get_install_dir(), "bin", "dse") args = [dse, "spark"] args += spark_options p = subprocess.Popen(args, env=env) p.wait()
def shuffle(self, cmd): cdir = self.get_cassandra_dir() shuffle = common.join_bin(cdir, 'bin', 'cassandra-shuffle') host = self.address() args = [ shuffle, '-h', host, '-p', str(self.jmx_port) ] + [ cmd ] try: subprocess.call(args) except KeyboardInterrupt: pass
def dsetool(self, cmd): env = self.get_env() extension.append_to_client_env(self, env) node_ip, binary_port = self.network_interfaces['binary'] dsetool = common.join_bin(self.get_install_dir(), 'bin', 'dsetool') args = [dsetool, '-h', node_ip, '-j', str(self.jmx_port), '-c', str(binary_port)] args += cmd.split() p = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return handle_external_tool_process(p, args)
def nodetool(self, cmd): cdir = self.get_cassandra_dir() nodetool = common.join_bin(cdir, "bin", "nodetool") env = common.make_cassandra_env(cdir, self.get_path()) host = self.address() args = [nodetool, "-h", host, "-p", str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def shuffle(self, cmd): cdir = self.get_cassandra_dir() shuffle = common.join_bin(cdir, "bin", "cassandra-shuffle") host = self.address() args = [shuffle, "-h", host, "-p", str(self.jmx_port)] + [cmd] try: subprocess.call(args) except KeyboardInterrupt: pass
def run_sstablesplit(self, datafile=None, size=None, keyspace=None, column_families=None): cdir = self.get_cassandra_dir() if self.cluster.version() >= "2.1": sstablesplit = common.join_bin(cdir, os.path.join('tools', 'bin'), 'sstablesplit') else: sstablesplit = common.join_bin(cdir, 'bin', 'sstablesplit') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile, keyspace, column_families) def do_split(f): print_("-- {0}-----".format(os.path.basename(f))) if size is not None: subprocess.call( [sstablesplit, '-s', str(size), f], cwd=os.path.join(cdir, 'bin'), env=env ) else: subprocess.call( [sstablesplit, f], cwd=os.path.join(cdir, 'bin'), env=env ) for datafile in datafiles: do_split(datafile)
def nodetool(self, cmd): cdir = self.get_cassandra_dir() nodetool = common.join_bin(cdir, 'bin', 'nodetool') env = common.make_cassandra_env(cdir, self.get_path()) host = self.address() args = [ nodetool, '-h', host, '-p', str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def spark(self, spark_options=None): if spark_options is None: spark_options = [] env = self.get_env() env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'spark'] args += spark_options p = subprocess.Popen(args, env=env) #Don't redirect stdout/stderr, users need to interact with new process return handle_external_tool_process(p, args)
def sqoop(self, sqoop_options=None): if sqoop_options is None: sqoop_options = [] env = self.get_env() env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'sqoop'] args += sqoop_options p = subprocess.Popen(args, env=env) p.wait()
def pig(self, pig_options=None): if pig_options is None: pig_options = [] env = common.make_dse_env(self.get_install_dir(), self.get_path()) env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'pig'] args += pig_options p = subprocess.Popen(args, env=env) p.wait()
def spark(self, spark_options=None): if spark_options is None: spark_options = [] env = common.make_dse_env(self.get_install_dir(), self.get_path()) env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'spark'] args += spark_options p = subprocess.Popen(args, env=env) p.wait()
def spark(self, spark_options=None): if spark_options is None: spark_options = [] env = self.get_env() env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'spark'] args += spark_options p = subprocess.Popen(args, env=env) p.wait()
def hadoop(self, hadoop_options=None): if hadoop_options is None: hadoop_options = [] env = self.get_env() env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse, 'hadoop'] args += hadoop_options p = subprocess.Popen(args, env=env) p.wait()
def sqoop(self, sqoop_options=None): if sqoop_options is None: sqoop_options = [] env = common.make_dse_env(self.get_install_dir(), self.get_path()) env["JMX_PORT"] = self.jmx_port dse = common.join_bin(self.get_install_dir(), "bin", "dse") args = [dse, "sqoop"] args += sqoop_options p = subprocess.Popen(args, env=env) p.wait()
def dse(self, dse_options=None): if dse_options is None: dse_options = [] env = self.get_env() extension.append_to_client_env(self, env) env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse] args += dse_options p = subprocess.Popen(args, env=env) #Don't redirect stdout/stderr, users need to interact with new process return handle_external_tool_process(p, args)
def dse(self, dse_options=None): if dse_options is None: dse_options = [] env = self.get_env() extension.append_to_client_env(self, env) env['JMX_PORT'] = self.jmx_port dse = common.join_bin(self.get_install_dir(), 'bin', 'dse') args = [dse] args += dse_options p = subprocess.Popen(args, env=env) p.wait()
def cli(self): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, "bin", "cassandra-cli") env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces["thrift"][0] port = self.network_interfaces["thrift"][1] args = ["-h", host, "-p", str(port), "--jmxport", str(self.jmx_port)] return CliSession( subprocess.Popen( [cli] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE ) )
def load(self, options): for itf in self.network_interfaces.values(): if itf: common.check_socket_available(itf) cdir = self.get_cassandra_dir() loader_bin = common.join_bin(cdir, "bin", "sstableloader") env = common.make_cassandra_env(cdir, self.get_path()) if not "-d" in options: l = [node.network_interfaces["storage"][0] for node in self.cluster.nodes.values() if node.is_live()] options = ["-d", ",".join(l)] + options # print "Executing with", options os.execve(loader_bin, [common.platform_binary("sstableloader")] + options, env)
def run_sstable2json(self, keyspace=None, datafile=None, column_families=None, enumerate_keys=False): cdir = self.get_cassandra_dir() sstable2json = common.join_bin(cdir, "bin", "sstable2json") env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile, keyspace, column_families) for file in datafiles: print_("-- {0} -----".format(os.path.basename(file))) args = [sstable2json, file] if enumerate_keys: args = args + ["-e"] subprocess.call(args, env=env) print_("")
def load(self, options): for itf in self.network_interfaces.values(): if itf: common.check_socket_available(itf) cdir = self.get_cassandra_dir() loader_bin = common.join_bin(cdir, 'bin', 'sstableloader') env = common.make_cassandra_env(cdir, self.get_path()) if not "-d" in options: l = [ node.network_interfaces['storage'][0] for node in self.cluster.nodes.values() if node.is_live() ] options = [ "-d", ",".join(l) ] + options #print "Executing with", options os.execve(loader_bin, [ common.platform_binary('sstableloader') ] + options, env)
def stop_win(self, wait=True, wait_other_notice=False, gently=True): # Gentle on Windows is relative. WM_CLOSE is the best we get without external scripting # New stop-server.bat allows for gentle shutdown on windows. If gentle shutdown # does not succeed for any reason, we will revert to the more forceful taskkill. # This is necessary intermittently. if self.cluster.version() >= "2.1": cass_bin = common.join_bin(self.get_path(), 'bin', 'stop-server') pidfile = os.path.join(self.get_path(), 'cassandra.pid') args = [ cass_bin, '-p', pidfile] if not gently: args.append('-f') proc = subprocess.Popen(args, cwd= self.get_bin_dir(), shell=True, stdout=subprocess.PIPE) pidfile = self.get_path() + "/cassandra.pid" if (os.path.isfile(pidfile)): os.remove(pidfile) still_running = self.is_running() if still_running and wait: wait_time_sec = 1 for i in xrange(0, 4): time.sleep(wait_time_sec) if not self.is_running(): return True wait_time_sec = wait_time_sec * 2 else: return True if gently: os.system("taskkill /PID " + str(self.pid)) else: os.system("taskkill /F /PID " + str(self.pid)) # no graceful shutdown on windows means it should be immediate cmd = 'tasklist /fi "PID eq ' + str(self.pid) + '"' proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) found = False for line in proc.stdout: if re.match("Image", line): found = True if found: return False else: pidfile = self.get_path() + "/cassandra.pid" if (os.path.isfile(pidfile)): os.remove(pidfile) return True
def stop_win(self, wait=True, wait_other_notice=False, gently=True): # Gentle on Windows is relative. WM_CLOSE is the best we get without external scripting # New stop-server.bat allows for gentle shutdown on windows. If gentle shutdown # does not succeed for any reason, we will revert to the more forceful taskkill. # This is necessary intermittently. if self.cluster.version() >= "2.1": cass_bin = common.join_bin(self.get_path(), "bin", "stop-server") pidfile = os.path.join(self.get_path(), "cassandra.pid") args = [cass_bin, "-p", pidfile] if not gently: args.append("-f") proc = subprocess.Popen(args, cwd=self.get_bin_dir(), shell=True, stdout=subprocess.PIPE) pidfile = self.get_path() + "/cassandra.pid" if os.path.isfile(pidfile): os.remove(pidfile) still_running = self.is_running() if still_running and wait: wait_time_sec = 1 for i in xrange(0, 4): time.sleep(wait_time_sec) if not self.is_running(): return True wait_time_sec = wait_time_sec * 2 else: return True if gently: os.system("taskkill /PID " + str(self.pid)) else: os.system("taskkill /F /PID " + str(self.pid)) # no graceful shutdown on windows means it should be immediate cmd = 'tasklist /fi "PID eq ' + str(self.pid) + '"' proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) found = False for line in proc.stdout: if re.match("Image", line): found = True if found: return False else: pidfile = self.get_path() + "/cassandra.pid" if os.path.isfile(pidfile): os.remove(pidfile) return True
def run_sstablesplit(self, datafile=None, size=None, keyspace=None, column_families=None): cdir = self.get_cassandra_dir() sstablesplit = common.join_bin(cdir, "bin", "sstablesplit") env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile, keyspace, column_families) def do_split(f): print_("-- {0}-----".format(os.path.basename(f))) if size is not None: subprocess.call([sstablesplit, "-s", str(size), f], cwd=os.path.join(cdir, "bin"), env=env) else: subprocess.call([sstablesplit, f], cwd=os.path.join(cdir, "bin"), env=env) for datafile in datafiles: do_split(datafile)
def run_sstable2json(self, keyspace=None, datafile=None, column_families=None, enumerate_keys=False): cdir = self.get_cassandra_dir() sstable2json = common.join_bin(cdir, 'bin', 'sstable2json') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile, keyspace, column_families) for file in datafiles: print_("-- {0} -----".format(os.path.basename(file))) args = [sstable2json, file] if enumerate_keys: args = args + ["-e"] subprocess.call(args, env=env) print_("")
def generate_dc_tokens(self, node_count, tokens): if self.cassandra_version() < '4' or ( self.partitioner and not ('Murmur3' in self.partitioner or 'Random' in self.partitioner)): raise common.ArgumentError( "generate-tokens script only for >=4.0 and Murmur3 or Random") if not ('num_tokens' in self._config_options and self._config_options['num_tokens'] is not None and int(self._config_options['num_tokens']) > 1): raise common.ArgumentError( "Cannot use generate-tokens script without num_tokens > 1") partitioner = 'RandomPartitioner' if ( self.partitioner and 'Random' in self.partitioner) else 'Murmur3Partitioner' generate_tokens = common.join_bin(self.get_install_dir(), os.path.join('tools', 'bin'), 'generatetokens') cmd_list = [ generate_tokens, '-n', str(node_count), '-t', str(self._config_options.get("num_tokens")), '--rf', str(min(3, node_count)), '-p', partitioner ] process = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ.copy()) # the first line is "Generating tokens for X nodes with" and can be ignored process.stdout.readline() for n in range(1, node_count + 1): stdout_output = re.sub(r'^.*?:', '', process.stdout.readline().decode("utf-8")) node_tokens = stdout_output.replace('[', '').replace(' ', '').replace( ']', '').replace('\n', '') tokens.append(node_tokens) common.debug("pregenerated tokens from cmd_list: {} are {}".format( str(cmd_list), tokens))
def start(self, join_ring=True, no_wait=False, verbose=False, update_pid=True, wait_other_notice=False, replace_token=None, replace_address=None, jvm_args=[], wait_for_binary_proto=False, profile_options=None, use_jna=False): """ Start the node. Options includes: - join_ring: if false, start the node with -Dcassandra.join_ring=False - no_wait: by default, this method returns when the node is started and listening to clients. If no_wait=True, the method returns sooner. - wait_other_notice: if True, this method returns only when all other live node of the cluster have marked this node UP. - replace_token: start the node with the -Dcassandra.replace_token option. - replace_address: start the node with the -Dcassandra.replace_address option. """ if self.is_running(): raise NodeError("%s is already running" % self.name) for itf in list(self.network_interfaces.values()): if itf is not None and replace_address is None: common.check_socket_available(itf) if wait_other_notice: marks = [ (node, node.mark_log()) for node in list(self.cluster.nodes.values()) if node.is_running() ] cdir = self.get_install_dir() launch_bin = common.join_bin(cdir, 'bin', 'dse') # Copy back the dse scripts since profiling may have modified it the previous time shutil.copy(launch_bin, self.get_bin_dir()) launch_bin = common.join_bin(self.get_path(), 'bin', 'dse') # If Windows, change entries in .bat file to split conf from binaries if common.is_win(): self.__clean_bat() if profile_options is not None: config = common.get_config() if not 'yourkit_agent' in config: raise NodeError("Cannot enable profile. You need to set 'yourkit_agent' to the path of your agent in a {0}/config".format(common.get_default_path_display_name())) cmd = '-agentpath:%s' % config['yourkit_agent'] if 'options' in profile_options: cmd = cmd + '=' + profile_options['options'] print_(cmd) # Yes, it's fragile as shit pattern=r'cassandra_parms="-Dlog4j.configuration=log4j-server.properties -Dlog4j.defaultInitOverride=true' common.replace_in_file(launch_bin, pattern, ' ' + pattern + ' ' + cmd + '"') os.chmod(launch_bin, os.stat(launch_bin).st_mode | stat.S_IEXEC) env = common.make_dse_env(self.get_install_dir(), self.get_path()) if common.is_win(): self._clean_win_jmx(); pidfile = os.path.join(self.get_path(), 'cassandra.pid') args = [launch_bin, 'cassandra'] if self.workload is not None: if 'hadoop' in self.workload: args.append('-t') if 'solr' in self.workload: args.append('-s') if 'spark' in self.workload: args.append('-k') if 'cfs' in self.workload: args.append('-c') args += [ '-p', pidfile, '-Dcassandra.join_ring=%s' % str(join_ring) ] if replace_token is not None: args.append('-Dcassandra.replace_token=%s' % str(replace_token)) if replace_address is not None: args.append('-Dcassandra.replace_address=%s' % str(replace_address)) if use_jna is False: args.append('-Dcassandra.boot_without_jna=true') args = args + jvm_args process = None if common.is_win(): # clean up any old dirty_pid files from prior runs if (os.path.isfile(self.get_path() + "/dirty_pid.tmp")): os.remove(self.get_path() + "/dirty_pid.tmp") process = subprocess.Popen(args, cwd=self.get_bin_dir(), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Our modified batch file writes a dirty output with more than just the pid - clean it to get in parity # with *nix operation here. if common.is_win(): self.__clean_win_pid() self._update_pid(process) elif update_pid: if no_wait: time.sleep(2) # waiting 2 seconds nevertheless to check for early errors and for the pid to be set else: for line in process.stdout: if verbose: print_(line.rstrip('\n')) self._update_pid(process) if not self.is_running(): raise NodeError("Error starting node %s" % self.name, process) if wait_other_notice: for node, mark in marks: node.watch_log_for_alive(self, from_mark=mark) if wait_for_binary_proto: self.watch_log_for("Starting listening for CQL clients") # we're probably fine at that point but just wait some tiny bit more because # the msg is logged just before starting the binary protocol server time.sleep(0.2) if self.cluster.hasOpscenter(): self._start_agent() return process
def get_tool_args(self, toolname): return [common.join_bin(os.path.join(self.get_install_dir(), 'resources', 'cassandra'), 'bin', 'dse'), toolname]
def get_launch_bin(self): cdir = self.get_install_dir() launch_bin = common.join_bin(cdir, 'bin', 'dse') # Copy back the dse scripts since profiling may have modified it the previous time shutil.copy(launch_bin, self.get_bin_dir()) return common.join_bin(self.get_path(), 'bin', 'dse')
def get_tool_args(self, toolname): return [ common.join_bin( os.path.join(self.get_install_dir(), 'resources', 'cassandra'), 'bin', 'dse'), toolname ]
def scrub(self, options): cdir = self.get_cassandra_dir() scrub_bin = common.join_bin(cdir, "bin", "sstablescrub") env = common.make_cassandra_env(cdir, self.get_path()) os.execve(scrub_bin, [common.platform_binary("sstablescrub")] + options, env)
def start(self, join_ring=True, no_wait=False, verbose=False, update_pid=True, wait_other_notice=False, replace_token=None, replace_address=None, jvm_args=None, wait_for_binary_proto=False, profile_options=None, use_jna=False, quiet_start=False): """ Start the node. Options includes: - join_ring: if false, start the node with -Dcassandra.join_ring=False - no_wait: by default, this method returns when the node is started and listening to clients. If no_wait=True, the method returns sooner. - wait_other_notice: if True, this method returns only when all other live node of the cluster have marked this node UP. - replace_token: start the node with the -Dcassandra.replace_token option. - replace_address: start the node with the -Dcassandra.replace_address option. """ if jvm_args is None: jvm_args = [] if self.is_running(): raise NodeError("%s is already running" % self.name) for itf in list(self.network_interfaces.values()): if itf is not None and replace_address is None: common.check_socket_available(itf) if wait_other_notice: marks = [(node, node.mark_log()) for node in list(self.cluster.nodes.values()) if node.is_running()] cdir = self.get_install_dir() launch_bin = common.join_bin(cdir, 'bin', 'dse') # Copy back the dse scripts since profiling may have modified it the previous time shutil.copy(launch_bin, self.get_bin_dir()) launch_bin = common.join_bin(self.get_path(), 'bin', 'dse') # If Windows, change entries in .bat file to split conf from binaries if common.is_win(): self.__clean_bat() if profile_options is not None: config = common.get_config() if 'yourkit_agent' not in config: raise NodeError( "Cannot enable profile. You need to set 'yourkit_agent' to the path of your agent in a {0}/config" .format(common.get_default_path_display_name())) cmd = '-agentpath:%s' % config['yourkit_agent'] if 'options' in profile_options: cmd = cmd + '=' + profile_options['options'] print_(cmd) # Yes, it's fragile as shit pattern = r'cassandra_parms="-Dlog4j.configuration=log4j-server.properties -Dlog4j.defaultInitOverride=true' common.replace_in_file(launch_bin, pattern, ' ' + pattern + ' ' + cmd + '"') os.chmod(launch_bin, os.stat(launch_bin).st_mode | stat.S_IEXEC) env = common.make_dse_env(self.get_install_dir(), self.get_path()) if common.is_win(): self._clean_win_jmx() pidfile = os.path.join(self.get_path(), 'cassandra.pid') args = [launch_bin, 'cassandra'] if self.workload is not None: if 'hadoop' in self.workload: args.append('-t') if 'solr' in self.workload: args.append('-s') if 'spark' in self.workload: args.append('-k') if 'cfs' in self.workload: args.append('-c') args += ['-p', pidfile, '-Dcassandra.join_ring=%s' % str(join_ring)] if replace_token is not None: args.append('-Dcassandra.replace_token=%s' % str(replace_token)) if replace_address is not None: args.append('-Dcassandra.replace_address=%s' % str(replace_address)) if use_jna is False: args.append('-Dcassandra.boot_without_jna=true') args = args + jvm_args process = None if common.is_win(): # clean up any old dirty_pid files from prior runs if (os.path.isfile(self.get_path() + "/dirty_pid.tmp")): os.remove(self.get_path() + "/dirty_pid.tmp") process = subprocess.Popen(args, cwd=self.get_bin_dir(), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Our modified batch file writes a dirty output with more than just the pid - clean it to get in parity # with *nix operation here. if common.is_win(): self.__clean_win_pid() self._update_pid(process) elif update_pid: if no_wait: time.sleep( 2 ) # waiting 2 seconds nevertheless to check for early errors and for the pid to be set else: for line in process.stdout: if verbose: print_(line.rstrip('\n')) self._update_pid(process) if not self.is_running(): raise NodeError("Error starting node %s" % self.name, process) if wait_other_notice: for node, mark in marks: node.watch_log_for_alive(self, from_mark=mark) if wait_for_binary_proto: self.wait_for_binary_interface() if self.cluster.hasOpscenter(): self._start_agent() return process
def scrub(self, options): cdir = self.get_cassandra_dir() scrub_bin = common.join_bin(cdir, 'bin', 'sstablescrub') env = common.make_cassandra_env(cdir, self.get_path()) os.execve(scrub_bin, [ common.platform_binary('sstablescrub') ] + options, env)
def get_tool_args(self, toolname): return [common.join_bin(os.path.join(self.get_install_dir(), "resources", "cassandra"), "bin", "dse"), toolname]
def get_tool(self, toolname): return common.join_bin(self.get_tool_java_dir(), 'bin', toolname)
def start(self, join_ring=True, no_wait=False, verbose=False, update_pid=True, wait_other_notice=False, replace_token=None, replace_address=None, jvm_args=None, wait_for_binary_proto=False, profile_options=None, use_jna=False, quiet_start=False): """ Start the node. Options includes: - join_ring: if false, start the node with -Dcassandra.join_ring=False - no_wait: by default, this method returns when the node is started and listening to clients. If no_wait=True, the method returns sooner. - wait_other_notice: if True, this method returns only when all other live node of the cluster have marked this node UP. - replace_token: start the node with the -Dcassandra.replace_token option. - replace_address: start the node with the -Dcassandra.replace_address option. """ if jvm_args is None: jvm_args = [] scylla_cassandra_mapping = { '-Dcassandra.replace_address_first_boot': '--replace-address-first-boot' } # Replace args in the form # ['-Dcassandra.foo=bar'] to ['-Dcassandra.foo', 'bar'] translated_args = [] new_jvm_args = [] for jvm_arg in jvm_args: if '=' in jvm_arg: split_option = jvm_arg.split("=") e_msg = ("Option %s not in the form '-Dcassandra.foo=bar'. " "Please check your test" % jvm_arg) assert len(split_option) == 2, e_msg option, value = split_option # If we have information on how to translate the jvm option, # translate it if option in scylla_cassandra_mapping: translated_args += [ scylla_cassandra_mapping[option], value ] # Otherwise, just pass it as is else: new_jvm_args.append(jvm_arg) else: new_jvm_args.append(jvm_arg) jvm_args = new_jvm_args if self.is_running(): raise NodeError("%s is already running" % self.name) for itf in list(self.network_interfaces.values()): if itf is not None and replace_address is None: try: common.check_socket_available(itf) except Exception as msg: print("{}. Looking for offending processes...".format(msg)) for proc in psutil.process_iter(): if any(self.cluster.ipprefix in cmd for cmd in proc.cmdline()): print("name={} pid={} cmdline={}".format( proc.name(), proc.pid, proc.cmdline())) raise msg marks = [] if wait_other_notice: marks = [(node, node.mark_log()) for node in list(self.cluster.nodes.values()) if node.is_running()] self.mark = self.mark_log() launch_bin = common.join_bin(self.get_path(), 'bin', 'scylla') options_file = os.path.join(self.get_path(), 'conf', 'scylla.yaml') os.chmod(launch_bin, os.stat(launch_bin).st_mode | stat.S_IEXEC) # TODO: we do not support forcing specific settings # TODO: workaround for api-address as we do not load it # from config file scylla#59 conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF) with open(conf_file, 'r') as f: data = yaml.safe_load(f) jvm_args = jvm_args + ['--api-address', data['api_address']] jvm_args = jvm_args + [ '--collectd-hostname', '%s.%s' % (socket.gethostname(), self.name) ] # Let's add jvm_args and the translated args args = [ launch_bin, '--options-file', options_file, '--log-to-stdout', '1' ] + jvm_args + translated_args # Lets search for default overrides in SCYLLA_EXT_OPTS scylla_ext_opts = os.getenv('SCYLLA_EXT_OPTS', "").split() opts_i = 0 orig_args = list(args) while opts_i < len(scylla_ext_opts): if scylla_ext_opts[opts_i].startswith("--scylla-manager="): opts_i += 1 elif scylla_ext_opts[opts_i].startswith('-'): add = False if scylla_ext_opts[opts_i] not in orig_args: add = True args.append(scylla_ext_opts[opts_i]) opts_i += 1 while opts_i < len(scylla_ext_opts) and not scylla_ext_opts[ opts_i].startswith('-'): if add: args.append(scylla_ext_opts[opts_i]) opts_i += 1 if '--developer-mode' not in args: args += ['--developer-mode', 'true'] if '--smp' not in args: # If --smp is not passed from cmdline, use default (--smp 1) args += ['--smp', str(self._smp)] elif self._smp_set_during_test: # If node.set_smp() is called during the test, ignore the --smp # passed from the cmdline. args[args.index('--smp') + 1] = str(self._smp) else: # Update self._smp based on command line parameter. # It may be used below, along with self._mem_mb_per_cpu, for calculating --memory self._smp = int(args[args.index('--smp') + 1]) if '--memory' not in args: # If --memory is not passed from cmdline, use default (512M per cpu) args += [ '--memory', '{}M'.format(self._mem_mb_per_cpu * self._smp) ] elif self._mem_set_during_test: # If node.set_mem_mb_per_cpu() is called during the test, ignore the --memory # passed from the cmdline. args[args.index('--memory') + 1] = '{}M'.format( self._mem_mb_per_cpu * self._smp) if '--default-log-level' not in args: args += ['--default-log-level', self.__global_log_level] # TODO add support for classes_log_level if '--collectd' not in args: args += ['--collectd', '0'] if '--cpuset' not in args: args += ['--overprovisioned'] if '--prometheus-address' not in args: args += ['--prometheus-address', data['api_address']] if replace_address: args += ['--replace-address', replace_address] args += ['--unsafe-bypass-fsync', '1'] scylla_process = self._start_scylla(args, marks, update_pid, wait_other_notice, wait_for_binary_proto) self._start_jmx(data) if not self._wait_java_up(data): e_msg = ( "Error starting node %s: unable to connect to scylla-jmx" % self.name) raise NodeError(e_msg, scylla_process) self.is_running() return scylla_process