def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# check for the cassandra-specific files
if opt.get('cassandra_config_file') is None:
print "ERROR: No cassandra_config_file configured. Aborting."
sys.exit(1)
if opt.get('keyspace_definitions_file') is None:
print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"
# test files
for key in ['cassandra_config_file', 'keyspace_definitions_file']:
if opt.get(key) is not None:
try:
url = urllib.urlopen(opt.get(key))
data = url.read()
except:
raise
print "The file defined by %s (%s) does not exist. Aborting." % (key, opt.get(key))
sys.exit(1)
# if self.service.get_instances() :
# print "This cluster is already running. It must be terminated prior to being launched again."
# sys.exit(1)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate(
(self.service.CASSANDRA_NODE,),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])
print "Launching cluster with %d instance(s)...please wait." % number_of_nodes
self.service.launch_cluster(instance_template,
opt.get('ssh_options'),
opt.get('cassandra_config_file'),
opt.get('keyspace_definitions_file'))
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_nodes, opt.get("instance_type"),
None, "cassandra")
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# check for the cassandra-specific files
if opt.get('cassandra_config_file') is None:
print "ERROR: No cassandra_config_file configured. Aborting."
sys.exit(1)
if opt.get('keyspace_definitions_file') is None:
print "WARNING: No keyspace_definitions_file configured. You can ignore this for Cassandra v0.6.x"
# test files
for key in ['cassandra_config_file', 'keyspace_definitions_file']:
if opt.get(key) is not None:
try:
url = urllib.urlopen(opt.get(key))
data = url.read()
except:
raise
print "The file defined by %s (%s) does not exist. Aborting." % (
key, opt.get(key))
sys.exit(1)
# if self.service.get_instances() :
# print "This cluster is already running. It must be terminated prior to being launched again."
# sys.exit(1)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate((self.service.CASSANDRA_NODE, ),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
instance_template.add_env_strings(
["CLUSTER_SIZE=%d" % number_of_nodes])
print "Launching cluster with %d instance(s)...please wait." % number_of_nodes
self.service.launch_cluster(instance_template, opt.get('ssh_options'),
opt.get('cassandra_config_file'),
opt.get('keyspace_definitions_file'))
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_nodes,
opt.get("instance_type"), None, "cassandra")
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_INSTANCES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
if self.service.get_instances() :
print "This cluster is already running. It must be terminated prior to being launched again."
sys.exit(1)
number_of_nodes = int(args[0])
instance_template = InstanceTemplate(
(self.service.CASSANDRA_NODE,),
number_of_nodes,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'))
instance_template.add_env_strings(["CLUSTER_SIZE=%d" % number_of_nodes])
print "Launching cluster with %d instance(s)...please wait." % number_of_nodes
self.service.launch_cluster(instance_template, opt)
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_nodes, opt.get("instance_type"),
None, "cassandra")
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_SLAVES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# if PROVIDER is set in the environment that takes precedence over
# anything in the clusters.cfg; hbase is the default if nothing is set
provider = os.environ.get("PROVIDER", opt.get("provider", "hbase")).lower()
# default for spot clusters is for the master to NOT be spot; munging
# some things around here if the opposite is specified
spot_cluster_orig = self._spot_config["spot_cluster"]
if spot_cluster_orig and self._spot_config["master_spot"]:
self._spot_config["spot_cluster"] = True
else:
self._spot_config["spot_cluster"] = False
number_of_slaves = int(args[0])
master_templates = [
InstanceTemplate(
(
self.service.NAMENODE,
self.service.SECONDARY_NAMENODE,
self.service.JOBTRACKER
),
1,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'),
self._spot_config) # don't want the master to be a spot instance
]
for it in master_templates:
it.add_env_strings([
"CLUSTER_SIZE=%d" % (number_of_slaves+1),
"PROVIDER=%s" % (provider)
])
print "Using %s as the backend datastore" % (provider)
print "Launching cluster with %d instance(s) - starting master...please wait." % (number_of_slaves+1)
master = self.service.launch_cluster(master_templates, opt.get('client_cidr'), opt.get('config_dir'))
if master is None:
print "An error occurred starting the master node. Check the logs for more information."
sys.exit(1)
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", 1, opt.get("instance_type"),
provider, "hadoop")
print "Master now running at %s - starting slaves" % master.public_dns_name
self._spot_config["spot_cluster"] = spot_cluster_orig
slave_templates = [
InstanceTemplate(
(
self.service.DATANODE,
self.service.TASKTRACKER
),
number_of_slaves,
opt.get('image_id'),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'),
opt.get('env'),
opt.get('security_groups'),
self._spot_config)
]
for it in slave_templates:
it.add_env_strings([
"CLUSTER_SIZE=%d" % (number_of_slaves+1),
"NN_HOST=%s" % master.public_dns_name,
"JT_HOST=%s" % master.public_dns_name,
"ZOOKEEPER_QUORUM=%s" % master.private_dns_name,
"PROVIDER=%s" % (provider)
])
print "Launching %d slave instance(s)...please wait." % (number_of_slaves)
slave = self.service.launch_cluster(slave_templates, opt.get('client_cidr'), opt.get('config_dir'))
if slave is None:
print "An error occurred starting the slave nodes. Check the logs for more details"
sys.exit(1)
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_slaves, opt.get("instance_type"),
provider, "hadoop")
#Once the cluster is up, if the provider is Cloudbase, we need to ensure that Cloudbase has been initialized
#and launch the servers
if provider == "cloudbase":
#log in to the master and run a startup script
print "Provider is cloudbase - starting cloudbase processes ... please wait"
self.service.start_cloudbase(options_dict,
options_dict.get("hadoop_user", "hadoop"),
options_dict.get("ssh_user", "root"))
print "Finished - browse the cluster at http://%s/" % master.public_dns_name
self.logger.debug("Startup complete.")
def terminate(self):
instances = self._get_instances(self._get_cluster_group_name(), "running")
if instances:
log_cluster_action(self.config_dir, self._get_cluster_group_name(),
"terminate-cluster", len(instances))
self.ec2Connection.terminate_instances([i.id for i in instances])
def launch_cluster(self, argv, options_dict):
"""
"""
expected_arguments = ["NUM_SLAVES"]
opt, args = self.parse_options(self._command_name,
argv,
expected_arguments=expected_arguments)
opt.update(options_dict)
# if PROVIDER is set in the environment that takes precedence over
# anything in the clusters.cfg; hbase is the default if nothing is set
provider = os.environ.get("PROVIDER", opt.get("provider",
"hbase")).lower()
number_of_slaves = int(args[0])
master_templates = [
InstanceTemplate(
(self.service.NAMENODE, self.service.SECONDARY_NAMENODE,
self.service.JOBTRACKER), 1, opt.get('image_id'),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups'))
]
for it in master_templates:
it.add_env_strings([
"CLUSTER_SIZE=%d" % (number_of_slaves + 1),
"PROVIDER=%s" % (provider)
])
print "Using %s as the backend datastore" % (provider)
print "Launching cluster with %d instance(s) - starting master...please wait." % (
number_of_slaves + 1)
master = self.service.launch_cluster(master_templates,
opt.get('client_cidr'),
opt.get('config_dir'))
if master is None:
print "An error occurred starting the master node. Check the logs for more information."
sys.exit(1)
log_cluster_action(opt.get('config_dir'),
self._cluster_name, "launch-cluster", 1,
opt.get("instance_type"), provider, "hadoop")
print "Master now running at %s - starting slaves" % master.public_dns_name
slave_templates = [
InstanceTemplate((self.service.DATANODE, self.service.TASKTRACKER),
number_of_slaves, opt.get('image_id'),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_groups'))
]
for it in slave_templates:
it.add_env_strings([
"CLUSTER_SIZE=%d" % (number_of_slaves + 1),
"NN_HOST=%s" % master.public_dns_name,
"JT_HOST=%s" % master.public_dns_name,
"ZOOKEEPER_QUORUM=%s" % master.private_dns_name,
"PROVIDER=%s" % (provider)
])
print "Launching %d slave instance(s)...please wait." % (
number_of_slaves)
slave = self.service.launch_cluster(slave_templates,
opt.get('client_cidr'),
opt.get('config_dir'))
if slave is None:
print "An error occurred starting the slave nodes. Check the logs for more details"
sys.exit(1)
log_cluster_action(opt.get('config_dir'), self._cluster_name,
"launch-cluster", number_of_slaves,
opt.get("instance_type"), provider, "hadoop")
#Once the cluster is up, if the provider is Cloudbase, we need to ensure that Cloudbase has been initialized
#and launch the servers
if provider == "cloudbase":
#log in to the master and run a startup script
print "Provider is cloudbase - starting cloudbase processes ... please wait"
self.service.start_cloudbase(
options_dict.get("ssh_options"), options_dict,
options_dict.get("hadoop_user", "hadoop"),
options_dict.get("ssh_user", "root"))
print "Finished - browse the cluster at http://%s/" % master.public_dns_name
self.logger.debug("Startup complete.")
def terminate(self):
instances = self._get_instances(self._get_cluster_group_name(), "running")
if instances:
log_cluster_action(self.config_dir, self._get_cluster_group_name(),
"terminate-cluster", len(instances))
self.ec2Connection.terminate_instances([i.id for i in instances])
