def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate(
(self.service.SIMPLE_NODE,),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])
print "Launching cluster with %d instance(s)...please wait." % number_of_nodes
self.service.launch_cluster(instance_template,
opt.get('ssh_options'),opt.get('wait_dir', '/'))
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# check for the cassandra-specific files
if opt.get('cassandra_config_file') is None:
print "ERROR: No cassandra_config_file configured. Aborting."
sys.exit(1)
if opt.get('keyspace_definitions_file') is None:
print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"
# test files
for key in ['cassandra_config_file', 'keyspace_definitions_file']:
if opt.get(key) is not None:
try:
url = urllib.urlopen(opt.get(key))
data = url.read()
except:
raise
print "The file defined by %s (%s) does not exist. Aborting." % (key, opt.get(key))
sys.exit(1)
# if self.service.get_instances() :
# print "This cluster is already running. It must be terminated prior to being launched again."
# sys.exit(1)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate(
(self.service.CASSANDRA_NODE,),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])
print "Launching cluster with %d instance(s)...please wait." % number_of_nodes
self.service.launch_cluster(instance_template,
opt.get('ssh_options'),
opt.get('cassandra_config_file'),
opt.get('keyspace_definitions_file'))
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_nodes, opt.get("instance_type"),
None, "cassandra")
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# check for the cassandra-specific files
if opt.get('cassandra_config_file') is None:
print "ERROR: No cassandra_config_file configured. Aborting."
sys.exit(1)
if opt.get('keyspace_definitions_file') is None:
print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"
# test files
for key in ['cassandra_config_file', 'keyspace_definitions_file']:
if opt.get(key) is not None:
try:
url = urllib.urlopen(opt.get(key))
data = url.read()
except:
raise
print "The file defined by %s (%s) does not exist. Aborting." % (
key, opt.get(key))
sys.exit(1)
# if self.service.get_instances() :
# print "This cluster is already running. It must be terminated prior to being launched again."
# sys.exit(1)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
instance_template.add_env_strings(
["CLUSTER_SIZE=%d" % number_of_nodes])
print "Launching cluster with %d instance(s)...please wait." % number_of_nodes
self.service.launch_cluster(instance_template, opt.get('ssh_options'),
opt.get('cassandra_config_file'),
opt.get('keyspace_definitions_file'))
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_nodes,
opt.get("instance_type"), None, "cassandra")
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
if self.service.get_instances() :
print "This cluster is already running. It must be terminated prior to being launched again."
sys.exit(1)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate(
(self.service.CASSANDRA_NODE,),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])
print "Launching cluster with %d instance(s)...please wait." % number_of_nodes
self.service.launch_cluster(instance_template, opt)
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_nodes, opt.get("instance_type"),
None, "cassandra")
def expand_cluster(self, argv, options_dict):
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments,
unbounded_args=True)
opt.update(options_dict)
# check for the cassandra-specific files
if opt.get('cassandra_config_file') is None:
print "ERROR: No cassandra_config_file configured. Aborting."
sys.exit(1)
# test files
for key in ['cassandra_config_file']:
if opt.get(key) is not None:
try:
url = urllib.urlopen(opt.get(key))
data = url.read()
except:
raise
print "The file defined by %s (%s) does not exist. Aborting." % (
key, opt.get(key))
sys.exit(1)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
# instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])
print "Expanding cluster by %d instance(s)...please wait." % number_of_nodes
self.service.expand_cluster(instance_template, opt.get('ssh_options'),
opt.get('cassandra_config_file'))
def replace_down_nodes(self, argv, options_dict):
opt, args = self.parse_options(self._command_name, argv)
opt.update(options_dict)
# check for the cassandra-specific files
if opt.get('cassandra_config_file') is None:
print "ERROR: No cassandra_config_file configured. Aborting."
sys.exit(1)
# test files
for key in ['cassandra_config_file']:
if opt.get(key) is not None:
try:
url = urllib.urlopen(opt.get(key))
data = url.read()
except:
raise
print "The file defined by %s (%s) does not exist. Aborting." % (
key, opt.get(key))
sys.exit(1)
number_of_nodes = len(
self.service.calc_down_nodes(opt.get('ssh_options')))
instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
# instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])
print "Replacing %d down instance(s)...please wait." % number_of_nodes
self.service.replace_down_nodes(instance_template,
opt.get('ssh_options'),
opt.get('cassandra_config_file'))
def launch_master(self, argv, options_dict):
"""Launch the master node of a CLUSTER."""
opt, args = self.parse_options(self._command_name, argv, BASIC_OPTIONS)
opt.update(options_dict)
provider = opt.get("provider")
if provider is None:
provider = "hbase"
else:
provider.lower()
instance_templates = [
InstanceTemplate(
(self.service.NAMENODE, self.service.SECONDARY_NAMENODE,
self.service.JOBTRACKER), 1, opt.get('image_id'),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups')),
]
for it in master_templates:
it.add_env_strings(["PROVIDER=%s" % (provider)])
print "Launching cluster master...please wait."
jobtracker = self.service.launch_cluster(instance_templates,
opt.get('client_cidr'),
opt.get('config_dir'))
if jobtracker is None:
print "An error occurred started the Hadoop service. Check the logs for more information."
sys.exit(1)
print "Browse the cluster at http://%s/" % jobtracker.public_dns_name
self.logger.debug("Startup complete.")
def launch_slaves(self, argv, options_dict):
"""Launch slave/datanodes in CLUSTER."""
expected_arguments = ["NUM_SLAVES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
provider = opt.get("provider")
if provider is None:
provider = "hbase"
else:
provider.lower()
try:
number_of_slaves = int(args[0])
except ValueError:
print("Number of slaves must be an integer")
return
instance_templates = [
InstanceTemplate((self.service.DATANODE, self.service.TASKTRACKER),
number_of_slaves, opt.get('image_id'),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups')),
]
# @todo - this is originally passed in when creating a cluster from
# scratch, need to figure out what to do if we're growing a cluster
#instance_template.add_env_strings([
# "CLUSTER_SIZE=%d" % (number_of_slaves+1)
#])
print("Launching %s slave%s for %s" %
(number_of_slaves, "" if number_of_slaves == 1 else "s",
self._cluster_name))
# this is needed to filter the jobtracker/namenode down into
# hadoop-site.xml for the new nodes
namenode = self.service.get_namenode()
jobtracker = self.service.get_jobtracker()
for instance_template in instance_templates:
instance_template.add_env_strings([
"NN_HOST=%s" % namenode.public_dns_name,
"JT_HOST=%s" % jobtracker.public_dns_name,
"ZOOKEEPER_QUORUM=%s" % namenode.private_dns_name,
"PROVIDER=%s" % (provider)
])
# I think this count can be wrong if run too soon after running
# terminate_dead_nodes
existing_tasktrackers = self.service.get_tasktrackers()
num_tasktrackers = len(
existing_tasktrackers) if existing_tasktrackers else 0
self.service.launch_cluster(instance_templates,
opt.get('client_cidr'),
opt.get('config_dir'),
num_existing_tasktrackers=num_tasktrackers)
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_SLAVES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# if PROVIDER is set in the environment that takes precedence over
# anything in the clusters.cfg; hbase is the default if nothing is set
provider = os.environ.get("PROVIDER", opt.get("provider",
"hbase")).lower()
number_of_slaves = int(args[0])
master_templates = [
InstanceTemplate(
(self.service.NAMENODE, self.service.SECONDARY_NAMENODE,
self.service.JOBTRACKER), 1, opt.get('image_id'),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups'))
]
for it in master_templates:
it.add_env_strings([
"CLUSTER_SIZE=%d" % (number_of_slaves + 1),
"PROVIDER=%s" % (provider)
])
print "Using %s as the backend datastore" % (provider)
print "Launching cluster with %d instance(s) - starting master...please wait." % (
number_of_slaves + 1)
master = self.service.launch_cluster(master_templates,
opt.get('client_cidr'),
opt.get('config_dir'))
if master is None:
print "An error occurred starting the master node. Check the logs for more information."
sys.exit(1)
log_cluster_action(opt.get('config_dir'),
self._cluster_name, "launch-cluster", 1,
opt.get("instance_type"), provider, "hadoop")
print "Master now running at %s - starting slaves" % master.public_dns_name
slave_templates = [
InstanceTemplate((self.service.DATANODE, self.service.TASKTRACKER),
number_of_slaves, opt.get('image_id'),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups'))
]
for it in slave_templates:
it.add_env_strings([
"CLUSTER_SIZE=%d" % (number_of_slaves + 1),
"NN_HOST=%s" % master.public_dns_name,
"JT_HOST=%s" % master.public_dns_name,
"ZOOKEEPER_QUORUM=%s" % master.private_dns_name,
"PROVIDER=%s" % (provider)
])
print "Launching %d slave instance(s)...please wait." % (
number_of_slaves)
slave = self.service.launch_cluster(slave_templates,
opt.get('client_cidr'),
opt.get('config_dir'))
if slave is None:
print "An error occurred starting the slave nodes. Check the logs for more details"
sys.exit(1)
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_slaves,
opt.get("instance_type"), provider, "hadoop")
#Once the cluster is up, if the provider is Cloudbase, we need to ensure that Cloudbase has been initialized
#and launch the servers
if provider == "cloudbase":
#log in to the master and run a startup script
print "Provider is cloudbase - starting cloudbase processes ... please wait"
self.service.start_cloudbase(
options_dict.get("ssh_options"), options_dict,
options_dict.get("hadoop_user", "hadoop"),
options_dict.get("ssh_user", "root"))
print "Finished - browse the cluster at http://%s/" % master.public_dns_name
self.logger.debug("Startup complete.")
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_SLAVES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# check for the cassandra-specific files
if opt.get('cassandra_config_file') is None:
print "ERROR: No cassandra_config_file configured. Aborting."
sys.exit(1)
if opt.get('keyspace_definitions_file') is None:
print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"
# test files
for key in ['cassandra_config_file', 'keyspace_definitions_file']:
if opt.get(key) is not None:
try:
url = urllib.urlopen(opt.get(key))
data = url.read()
except:
raise
print "The file defined by %s (%s) does not exist. Aborting." % (
key, opt.get(key))
sys.exit(1)
number_of_slaves = int(args[0])
instance_templates = [
InstanceTemplate((
self.service.NAMENODE,
self.service.SECONDARY_NAMENODE,
self.service.JOBTRACKER,
self.service.HADOOP_CASSANDRA_NODE,
), 1, opt.get('image_id'), opt.get('instance_type'),
opt.get('key_name'), opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups')),
InstanceTemplate((
self.service.DATANODE,
self.service.TASKTRACKER,
self.service.CASSANDRA_NODE,
), number_of_slaves, opt.get('image_id'), opt.get('instance_type'),
opt.get('key_name'), opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups'))
]
for it in instance_templates:
it.add_env_strings(["CLUSTER_SIZE=%d" % (number_of_slaves + 1)])
print "Launching cluster with %d instance(s)...please wait." % (
number_of_slaves + 1)
jobtracker = self.service.launch_cluster(
instance_templates, opt.get('client_cidr'), opt.get('config_dir'),
opt.get('ssh_options'), opt.get('cassandra_config_file'),
opt.get('keyspace_definitions_file'))
if jobtracker is None:
print "An error occurred started the Hadoop service. Check the logs for more information."
sys.exit(1)
print "Browse the cluster at http://%s/" % jobtracker.public_dns_name
self.logger.debug("Startup complete.")