def _get_default_parser(self, usage, description, ignore_unknown_options=False): if ignore_unknown_options: parser = ForgivingParser(usage=usage, description=description) else: parser = OptionParser(usage=usage, description=description) parser.add_option('--config-dir', type="string", dest="config_dir", help="Directory for the cluster files [default to {0}]".format(common.get_default_path_display_name())) return parser
def _get_default_parser(self, usage, description, ignore_unknown_options=False): if ignore_unknown_options: parser = ForgivingParser(usage=usage, description=description) else: parser = OptionParser(usage=usage, description=description) parser.add_option( '--config-dir', type="string", dest="config_dir", help="Directory for the cluster files [default to {0}]".format( common.get_default_path_display_name())) return parser
def start(self, join_ring=True, no_wait=False, verbose=False, update_pid=True, wait_other_notice=False, replace_token=None, replace_address=None, jvm_args=[], wait_for_binary_proto=False, profile_options=None, use_jna=False): """ Start the node. Options includes: - join_ring: if false, start the node with -Dcassandra.join_ring=False - no_wait: by default, this method returns when the node is started and listening to clients. If no_wait=True, the method returns sooner. - wait_other_notice: if True, this method returns only when all other live node of the cluster have marked this node UP. - replace_token: start the node with the -Dcassandra.replace_token option. - replace_address: start the node with the -Dcassandra.replace_address option. """ if self.is_running(): raise NodeError("%s is already running" % self.name) for itf in list(self.network_interfaces.values()): if itf is not None and replace_address is None: common.check_socket_available(itf) if wait_other_notice: marks = [ (node, node.mark_log()) for node in list(self.cluster.nodes.values()) if node.is_running() ] cdir = self.get_install_dir() launch_bin = common.join_bin(cdir, 'bin', 'dse') # Copy back the dse scripts since profiling may have modified it the previous time shutil.copy(launch_bin, self.get_bin_dir()) launch_bin = common.join_bin(self.get_path(), 'bin', 'dse') # If Windows, change entries in .bat file to split conf from binaries if common.is_win(): self.__clean_bat() if profile_options is not None: config = common.get_config() if not 'yourkit_agent' in config: raise NodeError("Cannot enable profile. You need to set 'yourkit_agent' to the path of your agent in a {0}/config".format(common.get_default_path_display_name())) cmd = '-agentpath:%s' % config['yourkit_agent'] if 'options' in profile_options: cmd = cmd + '=' + profile_options['options'] print_(cmd) # Yes, it's fragile as shit pattern=r'cassandra_parms="-Dlog4j.configuration=log4j-server.properties -Dlog4j.defaultInitOverride=true' common.replace_in_file(launch_bin, pattern, ' ' + pattern + ' ' + cmd + '"') os.chmod(launch_bin, os.stat(launch_bin).st_mode | stat.S_IEXEC) env = common.make_dse_env(self.get_install_dir(), self.get_path()) if common.is_win(): self._clean_win_jmx(); pidfile = os.path.join(self.get_path(), 'cassandra.pid') args = [launch_bin, 'cassandra'] if self.workload is not None: if 'hadoop' in self.workload: args.append('-t') if 'solr' in self.workload: args.append('-s') if 'spark' in self.workload: args.append('-k') if 'cfs' in self.workload: args.append('-c') args += [ '-p', pidfile, '-Dcassandra.join_ring=%s' % str(join_ring) ] if replace_token is not None: args.append('-Dcassandra.replace_token=%s' % str(replace_token)) if replace_address is not None: args.append('-Dcassandra.replace_address=%s' % str(replace_address)) if use_jna is False: args.append('-Dcassandra.boot_without_jna=true') args = args + jvm_args process = None if common.is_win(): # clean up any old dirty_pid files from prior runs if (os.path.isfile(self.get_path() + "/dirty_pid.tmp")): os.remove(self.get_path() + "/dirty_pid.tmp") process = subprocess.Popen(args, cwd=self.get_bin_dir(), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Our modified batch file writes a dirty output with more than just the pid - clean it to get in parity # with *nix operation here. if common.is_win(): self.__clean_win_pid() self._update_pid(process) elif update_pid: if no_wait: time.sleep(2) # waiting 2 seconds nevertheless to check for early errors and for the pid to be set else: for line in process.stdout: if verbose: print_(line.rstrip('\n')) self._update_pid(process) if not self.is_running(): raise NodeError("Error starting node %s" % self.name, process) if wait_other_notice: for node, mark in marks: node.watch_log_for_alive(self, from_mark=mark) if wait_for_binary_proto: self.watch_log_for("Starting listening for CQL clients") # we're probably fine at that point but just wait some tiny bit more because # the msg is logged just before starting the binary protocol server time.sleep(0.2) if self.cluster.hasOpscenter(): self._start_agent() return process
def start(self, join_ring=True, no_wait=False, verbose=False, update_pid=True, wait_other_notice=False, replace_token=None, replace_address=None, jvm_args=None, wait_for_binary_proto=False, profile_options=None, use_jna=False, quiet_start=False): """ Start the node. Options includes: - join_ring: if false, start the node with -Dcassandra.join_ring=False - no_wait: by default, this method returns when the node is started and listening to clients. If no_wait=True, the method returns sooner. - wait_other_notice: if True, this method returns only when all other live node of the cluster have marked this node UP. - replace_token: start the node with the -Dcassandra.replace_token option. - replace_address: start the node with the -Dcassandra.replace_address option. """ if jvm_args is None: jvm_args = [] if self.is_running(): raise NodeError("%s is already running" % self.name) for itf in list(self.network_interfaces.values()): if itf is not None and replace_address is None: common.check_socket_available(itf) if wait_other_notice: marks = [(node, node.mark_log()) for node in list(self.cluster.nodes.values()) if node.is_running()] cdir = self.get_install_dir() launch_bin = common.join_bin(cdir, 'bin', 'dse') # Copy back the dse scripts since profiling may have modified it the previous time shutil.copy(launch_bin, self.get_bin_dir()) launch_bin = common.join_bin(self.get_path(), 'bin', 'dse') # If Windows, change entries in .bat file to split conf from binaries if common.is_win(): self.__clean_bat() if profile_options is not None: config = common.get_config() if 'yourkit_agent' not in config: raise NodeError( "Cannot enable profile. You need to set 'yourkit_agent' to the path of your agent in a {0}/config" .format(common.get_default_path_display_name())) cmd = '-agentpath:%s' % config['yourkit_agent'] if 'options' in profile_options: cmd = cmd + '=' + profile_options['options'] print_(cmd) # Yes, it's fragile as shit pattern = r'cassandra_parms="-Dlog4j.configuration=log4j-server.properties -Dlog4j.defaultInitOverride=true' common.replace_in_file(launch_bin, pattern, ' ' + pattern + ' ' + cmd + '"') os.chmod(launch_bin, os.stat(launch_bin).st_mode | stat.S_IEXEC) env = common.make_dse_env(self.get_install_dir(), self.get_path()) if common.is_win(): self._clean_win_jmx() pidfile = os.path.join(self.get_path(), 'cassandra.pid') args = [launch_bin, 'cassandra'] if self.workload is not None: if 'hadoop' in self.workload: args.append('-t') if 'solr' in self.workload: args.append('-s') if 'spark' in self.workload: args.append('-k') if 'cfs' in self.workload: args.append('-c') args += ['-p', pidfile, '-Dcassandra.join_ring=%s' % str(join_ring)] if replace_token is not None: args.append('-Dcassandra.replace_token=%s' % str(replace_token)) if replace_address is not None: args.append('-Dcassandra.replace_address=%s' % str(replace_address)) if use_jna is False: args.append('-Dcassandra.boot_without_jna=true') args = args + jvm_args process = None if common.is_win(): # clean up any old dirty_pid files from prior runs if (os.path.isfile(self.get_path() + "/dirty_pid.tmp")): os.remove(self.get_path() + "/dirty_pid.tmp") process = subprocess.Popen(args, cwd=self.get_bin_dir(), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Our modified batch file writes a dirty output with more than just the pid - clean it to get in parity # with *nix operation here. if common.is_win(): self.__clean_win_pid() self._update_pid(process) elif update_pid: if no_wait: time.sleep( 2 ) # waiting 2 seconds nevertheless to check for early errors and for the pid to be set else: for line in process.stdout: if verbose: print_(line.rstrip('\n')) self._update_pid(process) if not self.is_running(): raise NodeError("Error starting node %s" % self.name, process) if wait_other_notice: for node, mark in marks: node.watch_log_for_alive(self, from_mark=mark) if wait_for_binary_proto: self.wait_for_binary_interface() if self.cluster.hasOpscenter(): self._start_agent() return process
def get_parser(self): usage = "usage: ccm create [options] cluster_name" parser = self._get_default_parser(usage, self.description()) parser.add_option('--no-switch', action="store_true", dest="no_switch", help="Don't switch to the newly created cluster", default=False) parser.add_option('-p', '--partitioner', type="string", dest="partitioner", help="Set the cluster partitioner class") parser.add_option('-v', "--version", type="string", dest="version", help="Download and use provided cassandra or dse version. If version is of the form 'git:<branch name>', then the specified cassandra branch will be downloaded from the git repo and compiled. (takes precedence over --install-dir)", default=None) parser.add_option('-o', "--opsc", type="string", dest="opscenter", help="Download and use provided opscenter version to install with DSE. Will have no effect on cassandra installs)", default=None) parser.add_option("--dse", action="store_true", dest="dse", help="Use with -v to indicate that the version being loaded is DSE") parser.add_option("--dse-username", type="string", dest="dse_username", help="The username to use to download DSE with", default=None) parser.add_option("--dse-password", type="string", dest="dse_password", help="The password to use to download DSE with", default=None) parser.add_option("--dse-credentials", type="string", dest="dse_credentials_file", help="An ini-style config file containing the dse_username and dse_password under a dse_credentials section. [default to {}/.dse.ini if it exists]".format(common.get_default_path_display_name()), default=None) parser.add_option("--install-dir", type="string", dest="install_dir", help="Path to the cassandra or dse directory to use [default %default]", default="./") parser.add_option('-n', '--nodes', type="string", dest="nodes", help="Populate the new cluster with that number of nodes (a single int or a colon-separate list of ints for multi-dc setups)") parser.add_option('-i', '--ipprefix', type="string", dest="ipprefix", help="Ipprefix to use to create the ip of a node while populating") parser.add_option('-I', '--ip-format', type="string", dest="ipformat", help="Format to use when creating the ip of a node (supports enumerating ipv6-type addresses like fe80::%d%lo0)") parser.add_option('-s', "--start", action="store_true", dest="start_nodes", help="Start nodes added through -s", default=False) parser.add_option('-d', "--debug", action="store_true", dest="debug", help="If -s is used, show the standard output when starting the nodes", default=False) parser.add_option('-b', "--binary-protocol", action="store_true", dest="binary_protocol", help="Enable the binary protocol (starting from C* 1.2.5 the binary protocol is started by default and this option is a no-op)", default=False) parser.add_option('-D', "--debug-log", action="store_true", dest="debug_log", help="With -n, sets debug logging on the new nodes", default=False) parser.add_option('-T', "--trace-log", action="store_true", dest="trace_log", help="With -n, sets trace logging on the new nodes", default=False) parser.add_option("--vnodes", action="store_true", dest="vnodes", help="Use vnodes (256 tokens). Must be paired with -n.", default=False) parser.add_option('--jvm_arg', action="append", dest="jvm_args", help="Specify a JVM argument", default=[]) parser.add_option('--profile', action="store_true", dest="profile", help="Start the nodes with yourkit agent (only valid with -s)", default=False) parser.add_option('--profile-opts', type="string", action="store", dest="profile_options", help="Yourkit options when profiling", default=None) parser.add_option('--ssl', type="string", dest="ssl_path", help="Path to keystore.jks and cassandra.crt files (and truststore.jks [not required])", default=None) parser.add_option('--require_client_auth', action="store_true", dest="require_client_auth", help="Enable client authentication (only vaid with --ssl)", default=False) parser.add_option('--node-ssl', type="string", dest="node_ssl_path", help="Path to keystore.jks and truststore.jks for internode encryption", default=None) parser.add_option('--pwd-auth', action="store_true", dest="node_pwd_auth", help="Change authenticator to PasswordAuthenticator (default credentials)", default=False) parser.add_option('--byteman', action="store_true", dest="install_byteman", help="Start nodes with byteman agent running", default=False) parser.add_option('--root', action="store_true", dest="allow_root", help="Allow CCM to start cassandra as root", default=False) parser.add_option('--datadirs', type="int", dest="datadirs", help="Number of data directories to use", default=1) return parser
def get_parser(self): usage = "usage: ccm create [options] cluster_name" parser = self._get_default_parser(usage, self.description()) parser.add_option('--no-switch', action="store_true", dest="no_switch", help="Don't switch to the newly created cluster", default=False) parser.add_option('-p', '--partitioner', type="string", dest="partitioner", help="Set the cluster partitioner class") parser.add_option('-v', "--version", type="string", dest="version", help="Download and use provided cassandra or dse version. If version is of the form 'git:<branch name>', then the specified cassandra branch will be downloaded from the git repo and compiled. (takes precedence over --install-dir)", default=None) parser.add_option('-o', "--opsc", type="string", dest="opscenter", help="Download and use provided opscenter version to install with DSE. Will have no effect on cassandra installs)", default=None) parser.add_option("--dse", action="store_true", dest="dse", help="Use with -v to indicate that the version being loaded is DSE") parser.add_option("--dse-username", type="string", dest="dse_username", help="The username to use to download DSE with", default=None) parser.add_option("--dse-password", type="string", dest="dse_password", help="The password to use to download DSE with", default=None) parser.add_option("--dse-credentials", type="string", dest="dse_credentials_file", help="An ini-style config file containing the dse_username and dse_password under a dse_credentials section. [default to {}/.dse.ini if it exists]".format(common.get_default_path_display_name()), default=None) parser.add_option("--install-dir", type="string", dest="install_dir", help="Path to the cassandra or dse directory to use [default %default]", default="./") parser.add_option('-n', '--nodes', type="string", dest="nodes", help="Populate the new cluster with that number of nodes (a single int or a colon-separate list of ints for multi-dc setups)") parser.add_option('-i', '--ipprefix', type="string", dest="ipprefix", help="Ipprefix to use to create the ip of a node while populating") parser.add_option('-I', '--ip-format', type="string", dest="ipformat", help="Format to use when creating the ip of a node (supports enumerating ipv6-type addresses like fe80::%d%lo0)") parser.add_option('-s', "--start", action="store_true", dest="start_nodes", help="Start nodes added through -s", default=False) parser.add_option('-d', "--debug", action="store_true", dest="debug", help="If -s is used, show the standard output when starting the nodes", default=False) parser.add_option('-b', "--binary-protocol", action="store_true", dest="binary_protocol", help="Enable the binary protocol (starting from C* 1.2.5 the binary protocol is started by default and this option is a no-op)", default=False) parser.add_option('-D', "--debug-log", action="store_true", dest="debug_log", help="With -n, sets debug logging on the new nodes", default=False) parser.add_option('-T', "--trace-log", action="store_true", dest="trace_log", help="With -n, sets trace logging on the new nodes", default=False) parser.add_option("--vnodes", action="store_true", dest="vnodes", help="Use vnodes (256 tokens). Must be paired with -n.", default=False) parser.add_option('--jvm_arg', action="append", dest="jvm_args", help="Specify a JVM argument", default=[]) parser.add_option('--profile', action="store_true", dest="profile", help="Start the nodes with yourkit agent (only valid with -s)", default=False) parser.add_option('--profile-opts', type="string", action="store", dest="profile_options", help="Yourkit options when profiling", default=None) parser.add_option('--ssl', type="string", dest="ssl_path", help="Path to keystore.jks and cassandra.crt files (and truststore.jks [not required])", default=None) parser.add_option('--require_client_auth', action="store_true", dest="require_client_auth", help="Enable client authentication (only vaid with --ssl)", default=False) parser.add_option('--node-ssl', type="string", dest="node_ssl_path", help="Path to keystore.jks and truststore.jks for internode encryption", default=None) parser.add_option('--root', action="store_true", dest="allow_root", help="Allow CCM to start cassandra as root", default=False) parser.add_option('--byteman', action="store_true", dest="install_byteman", help="Start nodes with byteman agent running", default=False) return parser
class ClusterCreateCmd(Cmd): options_list = [ (['--no-switch'], { 'action': "store_true", 'dest': "no_switch", 'help': "Don't switch to the newly created cluster", 'default': False }), (['-p', '--partitioner'], { 'type': "string", 'dest': "partitioner", 'help': "Set the cluster partitioner class" }), (['-v', "--version"], { 'type': "string", 'dest': "version", 'help': "Download and use provided cassandra or dse version. If version is of the form 'git:<branch name>', then the specified cassandra branch will be downloaded from the git repo and compiled. (takes precedence over --install-dir)", 'default': None }), (['-o', "--opsc"], { 'type': "string", 'dest': "opscenter", 'help': "Download and use provided opscenter version to install with DSE. Will have no effect on cassandra installs)", 'default': None }), (["--dse"], { 'action': "store_true", 'dest': "dse", 'help': "Use with -v to indicate that the version being loaded is DSE" }), (["--dse-username"], { 'type': "string", 'dest': "dse_username", 'help': "The username to use to download DSE with", 'default': None }), (["--dse-password"], { 'type': "string", 'dest': "dse_password", 'help': "The password to use to download DSE with", 'default': None }), (["--dse-credentials"], { 'type': "string", 'dest': "dse_credentials_file", 'help': "An ini-style config file containing the dse_username and dse_password under a dse_credentials section. [default to {}/.dse.ini if it exists]" .format(common.get_default_path_display_name()), 'default': None }), (["--install-dir"], { 'type': "string", 'dest': "install_dir", 'help': "Path to the cassandra or dse directory to use [default %default]", 'default': "./" }), (['-n', '--nodes'], { 'type': "string", 'dest': "nodes", 'help': "Populate the new cluster with that number of nodes (a single int or a colon-separate list of ints for multi-dc setups)" }), (['-i', '--ipprefix'], { 'type': "string", 'dest': "ipprefix", 'help': "Ipprefix to use to create the ip of a node while populating" }), (['-I', '--ip-format'], { 'type': "string", 'dest': "ipformat", 'help': "Format to use when creating the ip of a node (supports enumerating ipv6-type addresses like fe80::%d%lo0)" }), (['-s', "--start"], { 'action': "store_true", 'dest': "start_nodes", 'help': "Start nodes added through -s", 'default': False }), (['-d', "--debug"], { 'action': "store_true", 'dest': "debug", 'help': "If -s is used, show the standard output when starting the nodes", 'default': False }), (['-b', "--binary-protocol"], { 'action': "store_true", 'dest': "binary_protocol", 'help': "Enable the binary protocol (starting from C* 1.2.5 the binary protocol is started by default and this option is a no-op)", 'default': False }), (['-D', "--debug-log"], { 'action': "store_true", 'dest': "debug_log", 'help': "With -n, sets debug logging on the new nodes", 'default': False }), (['-T', "--trace-log"], { 'action': "store_true", 'dest': "trace_log", 'help': "With -n, sets trace logging on the new nodes", 'default': False }), (["--vnodes"], { 'action': "store_true", 'dest': "vnodes", 'help': "Use vnodes (256 tokens). Must be paired with -n.", 'default': False }), (['--jvm_arg'], { 'action': "append", 'dest': "jvm_args", 'help': "Specify a JVM argument", 'default': [] }), (['--profile'], { 'action': "store_true", 'dest': "profile", 'help': "Start the nodes with yourkit agent (only valid with -s)", 'default': False }), (['--profile-opts'], { 'type': "string", 'action': "store", 'dest': "profile_options", 'help': "Yourkit options when profiling", 'default': None }), (['--ssl'], { 'type': "string", 'dest': "ssl_path", 'help': "Path to keystore.jks and cassandra.crt files (and truststore.jks [not required])", 'default': None }), (['--require_client_auth'], { 'action': "store_true", 'dest': "require_client_auth", 'help': "Enable client authentication (only vaid with --ssl)", 'default': False }), (['--node-ssl'], { 'type': "string", 'dest': "node_ssl_path", 'help': "Path to keystore.jks and truststore.jks for internode encryption", 'default': None }), (['--pwd-auth'], { 'action': "store_true", 'dest': "node_pwd_auth", 'help': "Change authenticator to PasswordAuthenticator (default credentials)", 'default': False }), (['--byteman'], { 'action': "store_true", 'dest': "install_byteman", 'help': "Start nodes with byteman agent running", 'default': False }), (['--root'], { 'action': "store_true", 'dest': "allow_root", 'help': "Allow CCM to start cassandra as root", 'default': False }), (['--datadirs'], { 'type': "int", 'dest': "datadirs", 'help': "Number of data directories to use", 'default': 1 }), ] descr_text = "Create a new cluster" usage = "usage: ccm create [options] cluster_name" def validate(self, parser, options, args): Cmd.validate(self, parser, options, args, cluster_name=True) if options.ipprefix and options.ipformat: parser.print_help() parser.error("%s and %s may not be used together" % (parser.get_option('-i'), parser.get_option('-I'))) self.nodes = parse_populate_count(options.nodes) if self.options.vnodes and self.nodes is None: print_( "Can't set --vnodes if not populating cluster in this command." ) parser.print_help() exit(1) if not options.version: try: common.validate_install_dir(options.install_dir) except ArgumentError: parser.print_help() parser.error( "%s is not a valid cassandra directory. You must define a cassandra dir or version." % options.install_dir) if common.get_dse_version(options.install_dir) is not None: common.assert_jdk_valid_for_cassandra_version( common.get_dse_cassandra_version(options.install_dir)) else: common.assert_jdk_valid_for_cassandra_version( common.get_version_from_build(options.install_dir)) if common.is_win() and os.path.exists('c:\windows\system32\java.exe'): print_("""WARN: c:\windows\system32\java.exe exists. This may cause registry issues, and jre7 to be used, despite jdk8 being installed. """) def run(self): try: if self.options.dse or (not self.options.version and common.isDse( self.options.install_dir)): cluster = DseCluster( self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, dse_username=self.options.dse_username, dse_password=self.options.dse_password, dse_credentials_file=self.options.dse_credentials_file, opscenter=self.options.opscenter, verbose=True) else: cluster = Cluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, verbose=True) except OSError as e: import traceback print_('Cannot create cluster: %s\n%s' % (str(e), traceback.format_exc()), file=sys.stderr) exit(1) if self.options.partitioner: cluster.set_partitioner(self.options.partitioner) if cluster.cassandra_version() >= "1.2.5": self.options.binary_protocol = True if self.options.binary_protocol: cluster.set_configuration_options({'start_native_transport': True}) if cluster.cassandra_version() >= "1.2" and self.options.vnodes: cluster.set_configuration_options({'num_tokens': 256}) if not self.options.no_switch: common.switch_cluster(self.path, self.name) print_('Current cluster is now: %s' % self.name) if not (self.options.ipprefix or self.options.ipformat): self.options.ipformat = '127.0.0.%d' if self.options.ssl_path: cluster.enable_ssl(self.options.ssl_path, self.options.require_client_auth) if self.options.node_ssl_path: cluster.enable_internode_ssl(self.options.node_ssl_path) if self.options.node_pwd_auth: cluster.enable_pwd_auth() if self.options.datadirs: cluster.set_datadir_count(self.options.datadirs) if self.nodes is not None: try: if self.options.debug_log: cluster.set_log_level("DEBUG") if self.options.trace_log: cluster.set_log_level("TRACE") cluster.populate(self.nodes, self.options.debug, use_vnodes=self.options.vnodes, ipprefix=self.options.ipprefix, ipformat=self.options.ipformat, install_byteman=self.options.install_byteman) if self.options.start_nodes: profile_options = None if self.options.profile: profile_options = {} if self.options.profile_options: profile_options[ 'options'] = self.options.profile_options if cluster.start( verbose=self.options.debug, wait_for_binary_proto=self.options.binary_protocol, jvm_args=self.options.jvm_args, profile_options=profile_options, allow_root=self.options.allow_root) is None: details = "" if not self.options.debug_log: details = " (you can use --debug for more information)" print_( "Error starting nodes, see above for details%s" % details, file=sys.stderr) except common.ArgumentError as e: print_(str(e), file=sys.stderr) exit(1)