Esempio n. 1
0
    def launch_cluster(self, argv, options_dict):
        """
        """
        expected_arguments = ["NUM_INSTANCES"]
        opt, args = self.parse_options(self._command_name, 
                                      argv,
                                      expected_arguments=expected_arguments)
        opt.update(options_dict)

        number_of_nodes = int(args[0])
        instance_template = InstanceTemplate(
            (self.service.SIMPLE_NODE,),
            number_of_nodes,
            opt.get('image_id'),
            opt.get('instance_type'),
            opt.get('key_name'),
            opt.get('public_key'), 
            opt.get('user_data_file'),
            opt.get('availability_zone'), 
            opt.get('user_packages'),
            opt.get('auto_shutdown'), 
            opt.get('env'),
            opt.get('security_groups'))
        instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])

        print "Launching cluster with %d instance(s)...please wait." % number_of_nodes

        self.service.launch_cluster(instance_template,
                                    opt.get('ssh_options'),opt.get('wait_dir', '/'))
Esempio n. 2
0
    def launch_cluster(self, argv, options_dict):
        """
        """
        expected_arguments = ["NUM_INSTANCES"]
        opt, args = self.parse_options(self._command_name, 
                                      argv,
                                      expected_arguments=expected_arguments)
        opt.update(options_dict)

        # check for the cassandra-specific files
        if opt.get('cassandra_config_file') is None:
            print "ERROR: No cassandra_config_file configured. Aborting."
            sys.exit(1)

        if opt.get('keyspace_definitions_file') is None:
            print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"

        # test files
        for key in ['cassandra_config_file', 'keyspace_definitions_file']:
            if opt.get(key) is not None:
                try:
                    url = urllib.urlopen(opt.get(key))
                    data = url.read()
                except: 
                    raise
                    print "The file defined by %s (%s) does not exist. Aborting." % (key, opt.get(key))
                    sys.exit(1)

#        if self.service.get_instances() :
#            print "This cluster is already running.  It must be terminated prior to being launched again."
#            sys.exit(1)

        number_of_nodes = int(args[0])
        instance_template = InstanceTemplate(
            (self.service.CASSANDRA_NODE,), 
            number_of_nodes,
            opt.get('image_id'),
            opt.get('instance_type'),
            opt.get('key_name'),
            opt.get('public_key'), 
            opt.get('user_data_file'),
            opt.get('availability_zone'), 
            opt.get('user_packages'),
            opt.get('auto_shutdown'), 
            opt.get('env'),
            opt.get('security_groups'))
        instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])

        print "Launching cluster with %d instance(s)...please wait." % number_of_nodes

        self.service.launch_cluster(instance_template,
                                    opt.get('ssh_options'),
                                    opt.get('cassandra_config_file'),
                                    opt.get('keyspace_definitions_file')) 


        log_cluster_action(opt.get('config_dir'), self._cluster_name,
            "launch-cluster", number_of_nodes, opt.get("instance_type"),
            None, "cassandra")
Esempio n. 3
0
    def launch_cluster(self, argv, options_dict):
        """
        """
        expected_arguments = ["NUM_INSTANCES"]
        opt, args = self.parse_options(self._command_name,
                                       argv,
                                       expected_arguments=expected_arguments)
        opt.update(options_dict)

        # check for the cassandra-specific files
        if opt.get('cassandra_config_file') is None:
            print "ERROR: No cassandra_config_file configured. Aborting."
            sys.exit(1)

        if opt.get('keyspace_definitions_file') is None:
            print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"

        # test files
        for key in ['cassandra_config_file', 'keyspace_definitions_file']:
            if opt.get(key) is not None:
                try:
                    url = urllib.urlopen(opt.get(key))
                    data = url.read()
                except:
                    raise
                    print "The file defined by %s (%s) does not exist. Aborting." % (
                        key, opt.get(key))
                    sys.exit(1)

#        if self.service.get_instances() :
#            print "This cluster is already running.  It must be terminated prior to being launched again."
#            sys.exit(1)

        number_of_nodes = int(args[0])
        instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ),
                                             number_of_nodes,
                                             opt.get('image_id'),
                                             opt.get('instance_type'),
                                             opt.get('key_name'),
                                             opt.get('public_key'),
                                             opt.get('user_data_file'),
                                             opt.get('availability_zone'),
                                             opt.get('user_packages'),
                                             opt.get('auto_shutdown'),
                                             opt.get('env'),
                                             opt.get('security_groups'))
        instance_template.add_env_strings(
            ["CLUSTER_SIZE=%d" % number_of_nodes])

        print "Launching cluster with %d instance(s)...please wait." % number_of_nodes

        self.service.launch_cluster(instance_template, opt.get('ssh_options'),
                                    opt.get('cassandra_config_file'),
                                    opt.get('keyspace_definitions_file'))

        log_cluster_action(opt.get('config_dir'), self._cluster_name,
                           "launch-cluster", number_of_nodes,
                           opt.get("instance_type"), None, "cassandra")
Esempio n. 4
0
    def launch_cluster(self, argv, options_dict):
        """
        """
        expected_arguments = ["NUM_INSTANCES"]
        opt, args = self.parse_options(self._command_name, 
                                      argv,
                                      expected_arguments=expected_arguments)
        opt.update(options_dict)

        if self.service.get_instances() :
            print "This cluster is already running.  It must be terminated prior to being launched again."
            sys.exit(1)

        number_of_nodes = int(args[0])
        instance_template = InstanceTemplate(
            (self.service.CASSANDRA_NODE,), 
            number_of_nodes,
            opt.get('image_id'),
            opt.get('instance_type'),
            opt.get('key_name'),
            opt.get('public_key'), 
            opt.get('user_data_file'),
            opt.get('availability_zone'), 
            opt.get('user_packages'),
            opt.get('auto_shutdown'), 
            opt.get('env'),
            opt.get('security_groups'))
        instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])

        print "Launching cluster with %d instance(s)...please wait." % number_of_nodes

        self.service.launch_cluster(instance_template, opt)


        log_cluster_action(opt.get('config_dir'), self._cluster_name,
            "launch-cluster", number_of_nodes, opt.get("instance_type"),
            None, "cassandra")
Esempio n. 5
0
    def expand_cluster(self, argv, options_dict):
        expected_arguments = ["NUM_INSTANCES"]
        opt, args = self.parse_options(self._command_name,
                                       argv,
                                       expected_arguments=expected_arguments,
                                       unbounded_args=True)
        opt.update(options_dict)

        # check for the cassandra-specific files
        if opt.get('cassandra_config_file') is None:
            print "ERROR: No cassandra_config_file configured. Aborting."
            sys.exit(1)

        # test files
        for key in ['cassandra_config_file']:
            if opt.get(key) is not None:
                try:
                    url = urllib.urlopen(opt.get(key))
                    data = url.read()
                except:
                    raise
                    print "The file defined by %s (%s) does not exist. Aborting." % (
                        key, opt.get(key))
                    sys.exit(1)

        number_of_nodes = int(args[0])
        instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ),
                                             number_of_nodes,
                                             opt.get('image_id'),
                                             opt.get('instance_type'),
                                             opt.get('key_name'),
                                             opt.get('public_key'),
                                             opt.get('user_data_file'),
                                             opt.get('availability_zone'),
                                             opt.get('user_packages'),
                                             opt.get('auto_shutdown'),
                                             opt.get('env'),
                                             opt.get('security_groups'))
        #        instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])

        print "Expanding cluster by %d instance(s)...please wait." % number_of_nodes

        self.service.expand_cluster(instance_template, opt.get('ssh_options'),
                                    opt.get('cassandra_config_file'))
Esempio n. 6
0
    def replace_down_nodes(self, argv, options_dict):
        opt, args = self.parse_options(self._command_name, argv)
        opt.update(options_dict)

        # check for the cassandra-specific files
        if opt.get('cassandra_config_file') is None:
            print "ERROR: No cassandra_config_file configured. Aborting."
            sys.exit(1)

        # test files
        for key in ['cassandra_config_file']:
            if opt.get(key) is not None:
                try:
                    url = urllib.urlopen(opt.get(key))
                    data = url.read()
                except:
                    raise
                    print "The file defined by %s (%s) does not exist. Aborting." % (
                        key, opt.get(key))
                    sys.exit(1)

        number_of_nodes = len(
            self.service.calc_down_nodes(opt.get('ssh_options')))
        instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ),
                                             number_of_nodes,
                                             opt.get('image_id'),
                                             opt.get('instance_type'),
                                             opt.get('key_name'),
                                             opt.get('public_key'),
                                             opt.get('user_data_file'),
                                             opt.get('availability_zone'),
                                             opt.get('user_packages'),
                                             opt.get('auto_shutdown'),
                                             opt.get('env'),
                                             opt.get('security_groups'))
        #        instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])

        print "Replacing %d down instance(s)...please wait." % number_of_nodes

        self.service.replace_down_nodes(instance_template,
                                        opt.get('ssh_options'),
                                        opt.get('cassandra_config_file'))
Esempio n. 7
0
    def launch_master(self, argv, options_dict):
        """Launch the master node of a CLUSTER."""

        opt, args = self.parse_options(self._command_name, argv, BASIC_OPTIONS)
        opt.update(options_dict)

        provider = opt.get("provider")
        if provider is None:
            provider = "hbase"
        else:
            provider.lower()

        instance_templates = [
            InstanceTemplate(
                (self.service.NAMENODE, self.service.SECONDARY_NAMENODE,
                 self.service.JOBTRACKER), 1, opt.get('image_id'),
                opt.get('instance_type'), opt.get('key_name'),
                opt.get('public_key'), opt.get('user_data_file'),
                opt.get('availability_zone'), opt.get('user_packages'),
                opt.get('auto_shutdown'), opt.get('env'),
                opt.get('security_groups')),
        ]

        for it in master_templates:
            it.add_env_strings(["PROVIDER=%s" % (provider)])

        print "Launching cluster master...please wait."
        jobtracker = self.service.launch_cluster(instance_templates,
                                                 opt.get('client_cidr'),
                                                 opt.get('config_dir'))

        if jobtracker is None:
            print "An error occurred started the Hadoop service. Check the logs for more information."
            sys.exit(1)

        print "Browse the cluster at http://%s/" % jobtracker.public_dns_name
        self.logger.debug("Startup complete.")
Esempio n. 8
0
    def launch_slaves(self, argv, options_dict):
        """Launch slave/datanodes in CLUSTER."""

        expected_arguments = ["NUM_SLAVES"]
        opt, args = self.parse_options(self._command_name,
                                       argv,
                                       expected_arguments=expected_arguments)
        opt.update(options_dict)

        provider = opt.get("provider")
        if provider is None:
            provider = "hbase"
        else:
            provider.lower()

        try:
            number_of_slaves = int(args[0])
        except ValueError:
            print("Number of slaves must be an integer")
            return

        instance_templates = [
            InstanceTemplate((self.service.DATANODE, self.service.TASKTRACKER),
                             number_of_slaves, opt.get('image_id'),
                             opt.get('instance_type'), opt.get('key_name'),
                             opt.get('public_key'), opt.get('user_data_file'),
                             opt.get('availability_zone'),
                             opt.get('user_packages'),
                             opt.get('auto_shutdown'), opt.get('env'),
                             opt.get('security_groups')),
        ]

        # @todo - this is originally passed in when creating a cluster from
        # scratch, need to figure out what to do if we're growing a cluster
        #instance_template.add_env_strings([
        #    "CLUSTER_SIZE=%d" % (number_of_slaves+1)
        #])

        print("Launching %s slave%s for %s" %
              (number_of_slaves, "" if number_of_slaves == 1 else "s",
               self._cluster_name))

        # this is needed to filter the jobtracker/namenode down into
        # hadoop-site.xml for the new nodes
        namenode = self.service.get_namenode()
        jobtracker = self.service.get_jobtracker()
        for instance_template in instance_templates:
            instance_template.add_env_strings([
                "NN_HOST=%s" % namenode.public_dns_name,
                "JT_HOST=%s" % jobtracker.public_dns_name,
                "ZOOKEEPER_QUORUM=%s" % namenode.private_dns_name,
                "PROVIDER=%s" % (provider)
            ])

        # I think this count can be wrong if run too soon after running
        # terminate_dead_nodes
        existing_tasktrackers = self.service.get_tasktrackers()
        num_tasktrackers = len(
            existing_tasktrackers) if existing_tasktrackers else 0
        self.service.launch_cluster(instance_templates,
                                    opt.get('client_cidr'),
                                    opt.get('config_dir'),
                                    num_existing_tasktrackers=num_tasktrackers)
Esempio n. 9
0
    def launch_cluster(self, argv, options_dict):
        """
        """

        expected_arguments = ["NUM_SLAVES"]
        opt, args = self.parse_options(self._command_name,
                                       argv,
                                       expected_arguments=expected_arguments)
        opt.update(options_dict)

        # if PROVIDER is set in the environment that takes precedence over
        # anything in the clusters.cfg; hbase is the default if nothing is set
        provider = os.environ.get("PROVIDER", opt.get("provider",
                                                      "hbase")).lower()

        number_of_slaves = int(args[0])
        master_templates = [
            InstanceTemplate(
                (self.service.NAMENODE, self.service.SECONDARY_NAMENODE,
                 self.service.JOBTRACKER), 1, opt.get('image_id'),
                opt.get('instance_type'), opt.get('key_name'),
                opt.get('public_key'), opt.get('user_data_file'),
                opt.get('availability_zone'), opt.get('user_packages'),
                opt.get('auto_shutdown'), opt.get('env'),
                opt.get('security_groups'))
        ]
        for it in master_templates:
            it.add_env_strings([
                "CLUSTER_SIZE=%d" % (number_of_slaves + 1),
                "PROVIDER=%s" % (provider)
            ])

        print "Using %s as the backend datastore" % (provider)

        print "Launching cluster with %d instance(s) - starting master...please wait." % (
            number_of_slaves + 1)

        master = self.service.launch_cluster(master_templates,
                                             opt.get('client_cidr'),
                                             opt.get('config_dir'))

        if master is None:
            print "An error occurred starting the master node. Check the logs for more information."
            sys.exit(1)

        log_cluster_action(opt.get('config_dir'),
                           self._cluster_name, "launch-cluster", 1,
                           opt.get("instance_type"), provider, "hadoop")

        print "Master now running at %s - starting slaves" % master.public_dns_name

        slave_templates = [
            InstanceTemplate((self.service.DATANODE, self.service.TASKTRACKER),
                             number_of_slaves, opt.get('image_id'),
                             opt.get('instance_type'), opt.get('key_name'),
                             opt.get('public_key'), opt.get('user_data_file'),
                             opt.get('availability_zone'),
                             opt.get('user_packages'),
                             opt.get('auto_shutdown'), opt.get('env'),
                             opt.get('security_groups'))
        ]

        for it in slave_templates:
            it.add_env_strings([
                "CLUSTER_SIZE=%d" % (number_of_slaves + 1),
                "NN_HOST=%s" % master.public_dns_name,
                "JT_HOST=%s" % master.public_dns_name,
                "ZOOKEEPER_QUORUM=%s" % master.private_dns_name,
                "PROVIDER=%s" % (provider)
            ])

        print "Launching %d slave instance(s)...please wait." % (
            number_of_slaves)
        slave = self.service.launch_cluster(slave_templates,
                                            opt.get('client_cidr'),
                                            opt.get('config_dir'))

        if slave is None:
            print "An error occurred starting the slave nodes.  Check the logs for more details"
            sys.exit(1)

        log_cluster_action(opt.get('config_dir'), self._cluster_name,
                           "launch-cluster", number_of_slaves,
                           opt.get("instance_type"), provider, "hadoop")

        #Once the cluster is up, if the provider is Cloudbase, we need to ensure that Cloudbase has been initialized
        #and launch the servers
        if provider == "cloudbase":

            #log in to the master and run a startup script
            print "Provider is cloudbase - starting cloudbase processes ... please wait"
            self.service.start_cloudbase(
                options_dict.get("ssh_options"), options_dict,
                options_dict.get("hadoop_user", "hadoop"),
                options_dict.get("ssh_user", "root"))

        print "Finished - browse the cluster at http://%s/" % master.public_dns_name

        self.logger.debug("Startup complete.")
Esempio n. 10
0
    def launch_cluster(self, argv, options_dict):
        """
        """

        expected_arguments = ["NUM_SLAVES"]
        opt, args = self.parse_options(self._command_name,
                                       argv,
                                       expected_arguments=expected_arguments)
        opt.update(options_dict)

        # check for the cassandra-specific files
        if opt.get('cassandra_config_file') is None:
            print "ERROR: No cassandra_config_file configured. Aborting."
            sys.exit(1)

        if opt.get('keyspace_definitions_file') is None:
            print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"

        # test files
        for key in ['cassandra_config_file', 'keyspace_definitions_file']:
            if opt.get(key) is not None:
                try:
                    url = urllib.urlopen(opt.get(key))
                    data = url.read()
                except:
                    raise
                    print "The file defined by %s (%s) does not exist. Aborting." % (
                        key, opt.get(key))
                    sys.exit(1)

        number_of_slaves = int(args[0])
        instance_templates = [
            InstanceTemplate((
                self.service.NAMENODE,
                self.service.SECONDARY_NAMENODE,
                self.service.JOBTRACKER,
                self.service.HADOOP_CASSANDRA_NODE,
            ), 1, opt.get('image_id'), opt.get('instance_type'),
                             opt.get('key_name'), opt.get('public_key'),
                             opt.get('user_data_file'),
                             opt.get('availability_zone'),
                             opt.get('user_packages'),
                             opt.get('auto_shutdown'), opt.get('env'),
                             opt.get('security_groups')),
            InstanceTemplate((
                self.service.DATANODE,
                self.service.TASKTRACKER,
                self.service.CASSANDRA_NODE,
            ), number_of_slaves, opt.get('image_id'), opt.get('instance_type'),
                             opt.get('key_name'), opt.get('public_key'),
                             opt.get('user_data_file'),
                             opt.get('availability_zone'),
                             opt.get('user_packages'),
                             opt.get('auto_shutdown'), opt.get('env'),
                             opt.get('security_groups'))
        ]

        for it in instance_templates:
            it.add_env_strings(["CLUSTER_SIZE=%d" % (number_of_slaves + 1)])

        print "Launching cluster with %d instance(s)...please wait." % (
            number_of_slaves + 1)
        jobtracker = self.service.launch_cluster(
            instance_templates, opt.get('client_cidr'), opt.get('config_dir'),
            opt.get('ssh_options'), opt.get('cassandra_config_file'),
            opt.get('keyspace_definitions_file'))

        if jobtracker is None:
            print "An error occurred started the Hadoop service. Check the logs for more information."
            sys.exit(1)

        print "Browse the cluster at http://%s/" % jobtracker.public_dns_name
        self.logger.debug("Startup complete.")