def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cqlsh') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] port = self.network_interfaces['thrift'][1] args = cqlsh_options + [ host, str(port) ] sys.stdout.flush() if cmds is None: os.execve(cli, [ common.platform_binary('cqlsh') ] + args, env) else: p = subprocess.Popen([ cli ] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) for cmd in cmds.split(';'): p.stdin.write(cmd + ';\n') p.stdin.write("quit;\n") p.wait() for err in p.stderr: print_("(EE) ", err, end='') if show_output: i = 0 for log in p.stdout: # first four lines are not interesting if i >= 4: print_(log, end='') i = i + 1
def run_cqlsh(self, cmds=None, show_output=False, cqlsh_options=[]): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cqlsh') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] if self.cluster.version() >= "2.1": port = self.network_interfaces['binary'][1] else: port = self.network_interfaces['thrift'][1] args = cqlsh_options + [host, str(port)] sys.stdout.flush() if cmds is None: os.execve(cli, [common.platform_binary('cqlsh')] + args, env) else: p = subprocess.Popen([cli] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) for cmd in cmds.split(';'): p.stdin.write(cmd + ';\n') p.stdin.write("quit;\n") p.wait() for err in p.stderr: print "(EE) " + err, if show_output: i = 0 for log in p.stdout: # first four lines are not interesting if i >= 4: print log, i = i + 1
def cli(self): cdir = self.get_cassandra_dir() cli = common.join_bin(cdir, 'bin', 'cassandra-cli') env = common.make_cassandra_env(cdir, self.get_path()) host = self.network_interfaces['thrift'][0] port = self.network_interfaces['thrift'][1] args = [ '-h', host, '-p', str(port) , '--jmxport', str(self.jmx_port) ] return CliSession(subprocess.Popen([ cli ] + args, env=env, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE))
def nodetool(self, cmd): cdir = self.get_cassandra_dir() nodetool = common.join_bin(cdir, 'bin', 'nodetool') env = common.make_cassandra_env(cdir, self.get_path()) host = self.address() args = [nodetool, '-h', host, '-p', str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def shuffle(self, cmd): cdir = self.get_cassandra_dir() shuffle = common.join_bin(cdir, 'bin', 'cassandra-shuffle') host = self.address() args = [ shuffle, '-h', host, '-p', str(self.jmx_port) ] + [ cmd ] try: subprocess.call(args) except KeyboardInterrupt: pass
def nodetool(self, cmd): cdir = self.get_cassandra_dir() nodetool = common.join_bin(cdir, 'bin', 'nodetool') env = common.make_cassandra_env(cdir, self.get_path()) host = self.address() args = [ nodetool, '-h', host, '-p', str(self.jmx_port)] args += cmd.split() p = subprocess.Popen(args, env=env) p.wait()
def shuffle(self, cmd): cdir = self.get_cassandra_dir() shuffle = common.join_bin(cdir, 'bin', 'cassandra-shuffle') host = self.address() args = [shuffle, '-h', host, '-p', str(self.jmx_port)] + [cmd] try: subprocess.call(args) except KeyboardInterrupt: pass
def load(self, options): for itf in self.network_interfaces.values(): if itf: common.check_socket_available(itf) cdir = self.get_cassandra_dir() loader_bin = common.join_bin(cdir, 'bin', 'sstableloader') env = common.make_cassandra_env(cdir, self.get_path()) if not "-d" in options: l = [ node.network_interfaces['storage'][0] for node in self.cluster.nodes.values() if node.is_live() ] options = [ "-d", ",".join(l) ] + options #print "Executing with", options os.execve(loader_bin, [ common.platform_binary('sstableloader') ] + options, env)
def run_sstable2json(self, keyspace=None, datafile=None, column_families=None, enumerate_keys=False): cdir = self.get_cassandra_dir() sstable2json = common.join_bin(cdir, 'bin', 'sstable2json') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile,keyspace,column_families) for file in datafiles: print_("-- {0} -----".format(os.path.basename(file))) args = [ sstable2json , file ] if enumerate_keys: args = args + ["-e"] subprocess.call(args, env=env) print_("")
def run_sstable2json(self, keyspace=None, datafile=None, column_families=None, enumerate_keys=False): cdir = self.get_cassandra_dir() sstable2json = common.join_bin(cdir, 'bin', 'sstable2json') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile,keyspace,column_families) for file in datafiles: print "-- {0} -----".format(os.path.basename(file)) args = [ sstable2json , file ] if enumerate_keys: args = args + ["-e"] subprocess.call(args, env=env) print ""
def run_sstablesplit(self, datafile=None, size=None, keyspace=None, column_families=None): cdir = self.get_cassandra_dir() sstablesplit = common.join_bin(cdir, 'bin', 'sstablesplit') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile, keyspace, column_families) def do_split(f): print_("-- {0}-----".format(os.path.basename(f))) if size is not None: subprocess.call( [sstablesplit, '-s', str(size), f], cwd=os.path.join(cdir, 'bin'), env=env ) else: subprocess.call( [sstablesplit, f], cwd=os.path.join(cdir, 'bin'), env=env ) for datafile in datafiles: do_split(datafile)
def run_sstablesplit(self, datafile=None, size=None, keyspace=None, column_families=None): cdir = self.get_cassandra_dir() sstablesplit = common.join_bin(cdir, 'bin', 'sstablesplit') env = common.make_cassandra_env(cdir, self.get_path()) datafiles = self.__gather_sstables(datafile, keyspace, column_families) def do_split(f): print "-- {0}-----".format(os.path.basename(f)) if size is not None: subprocess.call( [sstablesplit, '-s', str(size), f], cwd=os.path.join(cdir, 'bin'), env=env ) else: subprocess.call( [sstablesplit, f], cwd=os.path.join(cdir, 'bin'), env=env ) for datafile in datafiles: do_split(datafile)
def load(self, options): for itf in self.network_interfaces.values(): if itf: common.check_socket_available(itf) cdir = self.get_cassandra_dir() loader_bin = common.join_bin(cdir, 'bin', 'sstableloader') env = common.make_cassandra_env(cdir, self.get_path()) if not "-d" in options: l = [ node.network_interfaces['storage'][0] for node in self.cluster.nodes.values() if node.is_live() ] options = ["-d", ",".join(l)] + options #print "Executing with", options os.execve(loader_bin, [common.platform_binary('sstableloader')] + options, env)
def start(self, join_ring=True, no_wait=False, verbose=False, update_pid=True, wait_other_notice=False, replace_token=None, replace_address=None, jvm_args=[], wait_for_binary_proto=False, profile_options=None, use_jna=False): """ Start the node. Options includes: - join_ring: if false, start the node with -Dcassandra.join_ring=False - no_wait: by default, this method returns when the node is started and listening to clients. If no_wait=True, the method returns sooner. - wait_other_notice: if True, this method returns only when all other live node of the cluster have marked this node UP. - replace_token: start the node with the -Dcassandra.replace_token option. - replace_address: start the node with the -Dcassandra.replace_address option. """ if self.is_running(): raise NodeError("%s is already running" % self.name) for itf in self.network_interfaces.values(): if itf is not None and replace_address is None: common.check_socket_available(itf) if wait_other_notice: marks = [(node, node.mark_log()) for node in self.cluster.nodes.values() if node.is_running()] cdir = self.get_cassandra_dir() cass_bin = common.join_bin(cdir, 'bin', 'cassandra') # Copy back the cassandra scripts since profiling may have modified it the previous time shutil.copy(cass_bin, self.get_bin_dir()) cass_bin = common.join_bin(self.get_path(), 'bin', 'cassandra') # If Windows, change entries in .bat file to split conf from binaries if common.is_win(): self.__clean_bat() if profile_options is not None: config = common.get_config() if not 'yourkit_agent' in config: raise NodeError( "Cannot enable profile. You need to set 'yourkit_agent' to the path of your agent in a ~/.ccm/config" ) cmd = '-agentpath:%s' % config['yourkit_agent'] if 'options' in profile_options: cmd = cmd + '=' + profile_options['options'] print cmd # Yes, it's fragile as shit pattern = r'cassandra_parms="-Dlog4j.configuration=log4j-server.properties -Dlog4j.defaultInitOverride=true' common.replace_in_file(cass_bin, pattern, ' ' + pattern + ' ' + cmd + '"') os.chmod(cass_bin, os.stat(cass_bin).st_mode | stat.S_IEXEC) env = common.make_cassandra_env(cdir, self.get_path()) pidfile = os.path.join(self.get_path(), 'cassandra.pid') args = [ cass_bin, '-p', pidfile, '-Dcassandra.join_ring=%s' % str(join_ring) ] if replace_token is not None: args.append('-Dcassandra.replace_token=%s' % str(replace_token)) if replace_address is not None: args.append('-Dcassandra.replace_address=%s' % str(replace_address)) if use_jna is False: args.append('-Dcassandra.boot_without_jna=true') args = args + jvm_args process = None if common.is_win(): # clean up any old dirty_pid files from prior runs if (os.path.isfile(self.get_path() + "/dirty_pid.tmp")): os.remove(self.get_path() + "/dirty_pid.tmp") process = subprocess.Popen(args, cwd=self.get_bin_dir(), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Our modified batch file writes a dirty output with more than just the pid - clean it to get in parity # with *nix operation here. if common.is_win(): self.__clean_win_pid() self._update_pid(process) elif update_pid: if no_wait: time.sleep( 2 ) # waiting 2 seconds nevertheless to check for early errors and for the pid to be set else: for line in process.stdout: if verbose: print line.rstrip('\n') self._update_pid(process) if not self.is_running(): raise NodeError("Error starting node %s" % self.name, process) if wait_other_notice: for node, mark in marks: node.watch_log_for_alive(self, from_mark=mark) if wait_for_binary_proto: self.watch_log_for("Starting listening for CQL clients") # we're probably fine at that point but just wait some tiny bit more because # the msg is logged just before starting the binary protocol server time.sleep(0.2) return process
def scrub(self, options): cdir = self.get_cassandra_dir() scrub_bin = common.join_bin(cdir, 'bin', 'sstablescrub') env = common.make_cassandra_env(cdir, self.get_path()) os.execve(scrub_bin, [ common.platform_binary('sstablescrub') ] + options, env)
def start(self, join_ring=True, no_wait=False, verbose=False, update_pid=True, wait_other_notice=False, replace_token=None, replace_address=None, jvm_args=[], wait_for_binary_proto=False, profile_options=None, use_jna=False): """ Start the node. Options includes: - join_ring: if false, start the node with -Dcassandra.join_ring=False - no_wait: by default, this method returns when the node is started and listening to clients. If no_wait=True, the method returns sooner. - wait_other_notice: if True, this method returns only when all other live node of the cluster have marked this node UP. - replace_token: start the node with the -Dcassandra.replace_token option. - replace_address: start the node with the -Dcassandra.replace_address option. """ if self.is_running(): raise NodeError("%s is already running" % self.name) for itf in list(self.network_interfaces.values()): if itf is not None and replace_address is None: common.check_socket_available(itf) if wait_other_notice: marks = [ (node, node.mark_log()) for node in list(self.cluster.nodes.values()) if node.is_running() ] cdir = self.get_cassandra_dir() cass_bin = common.join_bin(cdir, 'bin', 'cassandra') # Copy back the cassandra scripts since profiling may have modified it the previous time shutil.copy(cass_bin, self.get_bin_dir()) cass_bin = common.join_bin(self.get_path(), 'bin', 'cassandra') # If Windows, change entries in .bat file to split conf from binaries if common.is_win(): self.__clean_bat() if profile_options is not None: config = common.get_config() if not 'yourkit_agent' in config: raise NodeError("Cannot enable profile. You need to set 'yourkit_agent' to the path of your agent in a ~/.ccm/config") cmd = '-agentpath:%s' % config['yourkit_agent'] if 'options' in profile_options: cmd = cmd + '=' + profile_options['options'] print_(cmd) # Yes, it's fragile as shit pattern=r'cassandra_parms="-Dlog4j.configuration=log4j-server.properties -Dlog4j.defaultInitOverride=true' common.replace_in_file(cass_bin, pattern, ' ' + pattern + ' ' + cmd + '"') os.chmod(cass_bin, os.stat(cass_bin).st_mode | stat.S_IEXEC) env = common.make_cassandra_env(cdir, self.get_path()) pidfile = os.path.join(self.get_path(), 'cassandra.pid') args = [ cass_bin, '-p', pidfile, '-Dcassandra.join_ring=%s' % str(join_ring) ] if replace_token is not None: args.append('-Dcassandra.replace_token=%s' % str(replace_token)) if replace_address is not None: args.append('-Dcassandra.replace_address=%s' % str(replace_address)) if use_jna is False: args.append('-Dcassandra.boot_without_jna=true') args = args + jvm_args process = None if common.is_win(): process = subprocess.Popen(args, cwd=self.get_bin_dir(), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Our modified batch file writes a dirty output with more than just the pid - clean it to get in parity # with *nix operation here. if common.is_win(): # short delay could give us a false positive on a node being started as it could die and delete the pid file # after we check, however dtests have assumptions on how long the starting process takes. time.sleep(.5) self.__clean_win_pid() self._update_pid(process) elif update_pid: if no_wait: time.sleep(2) # waiting 2 seconds nevertheless to check for early errors and for the pid to be set else: for line in process.stdout: if verbose: print_(line.rstrip('\n')) self._update_pid(process) if not self.is_running(): raise NodeError("Error starting node %s" % self.name, process) if wait_other_notice: for node, mark in marks: node.watch_log_for_alive(self, from_mark=mark) if wait_for_binary_proto: self.watch_log_for("Starting listening for CQL clients") # we're probably fine at that point but just wait some tiny bit more because # the msg is logged just before starting the binary protocol server time.sleep(0.2) return process
def scrub(self, options): cdir = self.get_cassandra_dir() scrub_bin = common.join_bin(cdir, 'bin', 'sstablescrub') env = common.make_cassandra_env(cdir, self.get_path()) os.execve(scrub_bin, [common.platform_binary('sstablescrub')] + options, env)