def launch_cluster(self, argv, options_dict): """ """ expected_arguments = ["NUM_INSTANCES"] opt, args = self.parse_options(self._command_name, argv, expected_arguments=expected_arguments) opt.update(options_dict) # check for the cassandra-specific files if opt.get('cassandra_config_file') is None: print "ERROR: No cassandra_config_file configured. Aborting." sys.exit(1) if opt.get('keyspace_definitions_file') is None: print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x" # test files for key in ['cassandra_config_file', 'keyspace_definitions_file']: if opt.get(key) is not None: try: url = urllib.urlopen(opt.get(key)) data = url.read() except: raise print "The file defined by %s (%s) does not exist. Aborting." % (key, opt.get(key)) sys.exit(1) # if self.service.get_instances() : # print "This cluster is already running. It must be terminated prior to being launched again." # sys.exit(1) number_of_nodes = int(args[0]) instance_template = InstanceTemplate( (self.service.CASSANDRA_NODE,), number_of_nodes, opt.get('image_id'), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_groups')) instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes]) print "Launching cluster with %d instance(s)...please wait." % number_of_nodes self.service.launch_cluster(instance_template, opt.get('ssh_options'), opt.get('cassandra_config_file'), opt.get('keyspace_definitions_file')) log_cluster_action(opt.get('config_dir'), self._cluster_name, "launch-cluster", number_of_nodes, opt.get("instance_type"), None, "cassandra")
def launch_cluster(self, argv, options_dict): """ """ expected_arguments = ["NUM_INSTANCES"] opt, args = self.parse_options(self._command_name, argv, expected_arguments=expected_arguments) opt.update(options_dict) # check for the cassandra-specific files if opt.get('cassandra_config_file') is None: print "ERROR: No cassandra_config_file configured. Aborting." sys.exit(1) if opt.get('keyspace_definitions_file') is None: print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x" # test files for key in ['cassandra_config_file', 'keyspace_definitions_file']: if opt.get(key) is not None: try: url = urllib.urlopen(opt.get(key)) data = url.read() except: raise print "The file defined by %s (%s) does not exist. Aborting." % ( key, opt.get(key)) sys.exit(1) # if self.service.get_instances() : # print "This cluster is already running. It must be terminated prior to being launched again." # sys.exit(1) number_of_nodes = int(args[0]) instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ), number_of_nodes, opt.get('image_id'), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_groups')) instance_template.add_env_strings( ["CLUSTER_SIZE=%d" % number_of_nodes]) print "Launching cluster with %d instance(s)...please wait." % number_of_nodes self.service.launch_cluster(instance_template, opt.get('ssh_options'), opt.get('cassandra_config_file'), opt.get('keyspace_definitions_file')) log_cluster_action(opt.get('config_dir'), self._cluster_name, "launch-cluster", number_of_nodes, opt.get("instance_type"), None, "cassandra")
def launch_cluster(self, argv, options_dict): """ """ expected_arguments = ["NUM_INSTANCES"] opt, args = self.parse_options(self._command_name, argv, expected_arguments=expected_arguments) opt.update(options_dict) if self.service.get_instances() : print "This cluster is already running. It must be terminated prior to being launched again." sys.exit(1) number_of_nodes = int(args[0]) instance_template = InstanceTemplate( (self.service.CASSANDRA_NODE,), number_of_nodes, opt.get('image_id'), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_groups')) instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes]) print "Launching cluster with %d instance(s)...please wait." % number_of_nodes self.service.launch_cluster(instance_template, opt) log_cluster_action(opt.get('config_dir'), self._cluster_name, "launch-cluster", number_of_nodes, opt.get("instance_type"), None, "cassandra")
def launch_cluster(self, argv, options_dict): """ """ expected_arguments = ["NUM_SLAVES"] opt, args = self.parse_options(self._command_name, argv, expected_arguments=expected_arguments) opt.update(options_dict) # if PROVIDER is set in the environment that takes precedence over # anything in the clusters.cfg; hbase is the default if nothing is set provider = os.environ.get("PROVIDER", opt.get("provider", "hbase")).lower() # default for spot clusters is for the master to NOT be spot; munging # some things around here if the opposite is specified spot_cluster_orig = self._spot_config["spot_cluster"] if spot_cluster_orig and self._spot_config["master_spot"]: self._spot_config["spot_cluster"] = True else: self._spot_config["spot_cluster"] = False number_of_slaves = int(args[0]) master_templates = [ InstanceTemplate( ( self.service.NAMENODE, self.service.SECONDARY_NAMENODE, self.service.JOBTRACKER ), 1, opt.get('image_id'), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_groups'), self._spot_config) # don't want the master to be a spot instance ] for it in master_templates: it.add_env_strings([ "CLUSTER_SIZE=%d" % (number_of_slaves+1), "PROVIDER=%s" % (provider) ]) print "Using %s as the backend datastore" % (provider) print "Launching cluster with %d instance(s) - starting master...please wait." % (number_of_slaves+1) master = self.service.launch_cluster(master_templates, opt.get('client_cidr'), opt.get('config_dir')) if master is None: print "An error occurred starting the master node. Check the logs for more information." sys.exit(1) log_cluster_action(opt.get('config_dir'), self._cluster_name, "launch-cluster", 1, opt.get("instance_type"), provider, "hadoop") print "Master now running at %s - starting slaves" % master.public_dns_name self._spot_config["spot_cluster"] = spot_cluster_orig slave_templates = [ InstanceTemplate( ( self.service.DATANODE, self.service.TASKTRACKER ), number_of_slaves, opt.get('image_id'), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_groups'), self._spot_config) ] for it in slave_templates: it.add_env_strings([ "CLUSTER_SIZE=%d" % (number_of_slaves+1), "NN_HOST=%s" % master.public_dns_name, "JT_HOST=%s" % master.public_dns_name, "ZOOKEEPER_QUORUM=%s" % master.private_dns_name, "PROVIDER=%s" % (provider) ]) print "Launching %d slave instance(s)...please wait." % (number_of_slaves) slave = self.service.launch_cluster(slave_templates, opt.get('client_cidr'), opt.get('config_dir')) if slave is None: print "An error occurred starting the slave nodes. Check the logs for more details" sys.exit(1) log_cluster_action(opt.get('config_dir'), self._cluster_name, "launch-cluster", number_of_slaves, opt.get("instance_type"), provider, "hadoop") #Once the cluster is up, if the provider is Cloudbase, we need to ensure that Cloudbase has been initialized #and launch the servers if provider == "cloudbase": #log in to the master and run a startup script print "Provider is cloudbase - starting cloudbase processes ... please wait" self.service.start_cloudbase(options_dict, options_dict.get("hadoop_user", "hadoop"), options_dict.get("ssh_user", "root")) print "Finished - browse the cluster at http://%s/" % master.public_dns_name self.logger.debug("Startup complete.")
def terminate(self): instances = self._get_instances(self._get_cluster_group_name(), "running") if instances: log_cluster_action(self.config_dir, self._get_cluster_group_name(), "terminate-cluster", len(instances)) self.ec2Connection.terminate_instances([i.id for i in instances])
def launch_cluster(self, argv, options_dict): """ """ expected_arguments = ["NUM_SLAVES"] opt, args = self.parse_options(self._command_name, argv, expected_arguments=expected_arguments) opt.update(options_dict) # if PROVIDER is set in the environment that takes precedence over # anything in the clusters.cfg; hbase is the default if nothing is set provider = os.environ.get("PROVIDER", opt.get("provider", "hbase")).lower() number_of_slaves = int(args[0]) master_templates = [ InstanceTemplate( (self.service.NAMENODE, self.service.SECONDARY_NAMENODE, self.service.JOBTRACKER), 1, opt.get('image_id'), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_groups')) ] for it in master_templates: it.add_env_strings([ "CLUSTER_SIZE=%d" % (number_of_slaves + 1), "PROVIDER=%s" % (provider) ]) print "Using %s as the backend datastore" % (provider) print "Launching cluster with %d instance(s) - starting master...please wait." % ( number_of_slaves + 1) master = self.service.launch_cluster(master_templates, opt.get('client_cidr'), opt.get('config_dir')) if master is None: print "An error occurred starting the master node. Check the logs for more information." sys.exit(1) log_cluster_action(opt.get('config_dir'), self._cluster_name, "launch-cluster", 1, opt.get("instance_type"), provider, "hadoop") print "Master now running at %s - starting slaves" % master.public_dns_name slave_templates = [ InstanceTemplate((self.service.DATANODE, self.service.TASKTRACKER), number_of_slaves, opt.get('image_id'), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_groups')) ] for it in slave_templates: it.add_env_strings([ "CLUSTER_SIZE=%d" % (number_of_slaves + 1), "NN_HOST=%s" % master.public_dns_name, "JT_HOST=%s" % master.public_dns_name, "ZOOKEEPER_QUORUM=%s" % master.private_dns_name, "PROVIDER=%s" % (provider) ]) print "Launching %d slave instance(s)...please wait." % ( number_of_slaves) slave = self.service.launch_cluster(slave_templates, opt.get('client_cidr'), opt.get('config_dir')) if slave is None: print "An error occurred starting the slave nodes. Check the logs for more details" sys.exit(1) log_cluster_action(opt.get('config_dir'), self._cluster_name, "launch-cluster", number_of_slaves, opt.get("instance_type"), provider, "hadoop") #Once the cluster is up, if the provider is Cloudbase, we need to ensure that Cloudbase has been initialized #and launch the servers if provider == "cloudbase": #log in to the master and run a startup script print "Provider is cloudbase - starting cloudbase processes ... please wait" self.service.start_cloudbase( options_dict.get("ssh_options"), options_dict, options_dict.get("hadoop_user", "hadoop"), options_dict.get("ssh_user", "root")) print "Finished - browse the cluster at http://%s/" % master.public_dns_name self.logger.debug("Startup complete.")