def _transfer_config_files(self, ssh_options, config_file, keyspace_file=None,
instances=None):
"""
"""
if instances is None:
instances = self.get_instances()
self.logger.debug("Waiting for %d Cassandra instance(s) to install..." % len(instances))
for instance in instances:
self._wait_for_cassandra_install(instance, ssh_options)
self.logger.debug("Copying configuration files to %d Cassandra instances..." % len(instances))
seed_ips = [str(instance.private_dns_name) for instance in instances[:2]]
tokens = self._get_evenly_spaced_tokens_for_n_instances(len(instances))
# for each instance, generate a config file from the original file and upload it to
# the cluster node
for i in range(len(instances)):
local_file, remote_file = self._modify_config_file(instances[i], config_file, seed_ips, str(tokens[i]))
# Upload modified config file
scp_command = 'scp %s -r %s %s:/usr/local/apache-cassandra/conf/%s' % (xstr(ssh_options),
local_file, instances[i].public_dns_name, remote_file)
subprocess.call(scp_command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
# delete temporary file
os.unlink(local_file)
if keyspace_file:
keyspace_data = urllib.urlopen(keyspace_file).read()
fd, temp_keyspace_file = tempfile.mkstemp(prefix="keyspaces.txt_", text=True)
os.write(fd, keyspace_data)
os.close(fd)
self.logger.debug("Copying keyspace definition file to first Cassandra instance...")
# Upload keyspace definitions file
scp_command = 'scp %s -r %s %s:/usr/local/apache-cassandra/conf/keyspaces.txt' % \
(xstr(ssh_options), temp_keyspace_file, instances[0].public_dns_name)
subprocess.call(scp_command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
# remove temporary file
os.unlink(temp_keyspace_file)
def _get_standard_ssh_command(self, instance, ssh_options, remote_command=None):
"""
Returns the complete SSH command ready for execution on the instance.
"""
cmd = "ssh %s %s" % (xstr(ssh_options), instance.public_dns_name)
if remote_command is not None:
cmd += " '%s'" % remote_command
return cmd
def hack_config_for_multi_region(self, ssh_options, seeds):
instances = self.get_instances()
downloaded_file = "cassandra.yaml.downloaded"
for instance in instances:
# download config file
print "downloading config from %s" % instance.public_dns_name
scp_command = 'scp %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml %s' % (
xstr(ssh_options), instance.public_dns_name, downloaded_file)
subprocess.call(scp_command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
print "modifying config from %s" % instance.public_dns_name
yaml = parse_yaml(urllib.urlopen(downloaded_file))
yaml['seed_provider'][0]['parameters'][0]['seeds'] = seeds
yaml['listen_address'] = str(instance.public_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
yaml['broadcast_address'] = socket.gethostbyname(
str(instance.public_dns_name))
yaml[
'endpoint_snitch'] = 'org.apache.cassandra.locator.Ec2MultiRegionSnitch'
print "saving config from %s" % instance.public_dns_name
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_',
text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
#upload config file
print "uploading new config to %s" % instance.public_dns_name
scp_command = 'scp %s %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml' % (
xstr(ssh_options), temp_file, instance.public_dns_name)
subprocess.check_call(scp_command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
os.unlink(temp_file)
os.unlink(downloaded_file)
def _get_standard_ssh_command(self,
instance,
ssh_options,
remote_command=None,
username="******"):
"""
Returns the complete SSH command ready for execution on the instance.
"""
cmd = "ssh %s %s@%s" % (xstr(ssh_options), username,
instance.public_dns_name)
if remote_command is not None:
cmd += " '%s'" % remote_command
return cmd
def proxy(self, ssh_options, instances=None):
if instances is None:
return None
namenode = self.get_namenode()
if namenode is None:
self.logger.error("No namenode running. Aborting.")
return None
options = '-o "ConnectTimeout 10" -o "ServerAliveInterval 60" ' "-N -D 6666"
process = subprocess.Popen(
"ssh %s %s root@%s" % (xstr(ssh_options), options, namenode.public_dns_name),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
)
return process.pid
def proxy(self, ssh_options, instances=None):
if instances is None:
return None
namenode = self._get_namenode()
if namenode is None:
self.logger.error("No namenode running. Aborting.")
return None
options = '-o "ConnectTimeout 10" -o "ServerAliveInterval 60" ' \
'-N -D 6666'
process = subprocess.Popen(
'ssh %s %s root@%s' %
(xstr(ssh_options), options, namenode.public_dns_name),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
return process.pid
def _transfer_config_files(self,
ssh_options,
config_file,
keyspace_file=None,
instances=None):
"""
"""
if instances is None:
instances = self.get_instances()
self.logger.debug(
"Waiting for %d Cassandra instance(s) to install..." %
len(instances))
for instance in instances:
self._wait_for_cassandra_install(instance, ssh_options)
self.logger.debug(
"Copying configuration files to %d Cassandra instances..." %
len(instances))
seed_ips = [
str(instance.private_dns_name) for instance in instances[:2]
]
tokens = self._get_evenly_spaced_tokens_for_n_instances(len(instances))
# for each instance, generate a config file from the original file and upload it to
# the cluster node
for i in range(len(instances)):
local_file, remote_file = self._modify_config_file(
instances[i], config_file, seed_ips, str(tokens[i]))
# Upload modified config file
scp_command = 'scp %s -r %s root@%s:/usr/local/apache-cassandra/conf/%s' % (
xstr(ssh_options), local_file, instances[i].public_dns_name,
remote_file)
subprocess.call(scp_command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
# delete temporary file
os.unlink(local_file)
if keyspace_file:
keyspace_data = urllib.urlopen(keyspace_file).read()
fd, temp_keyspace_file = tempfile.mkstemp(prefix="keyspaces.txt_",
text=True)
os.write(fd, keyspace_data)
os.close(fd)
self.logger.debug(
"Copying keyspace definition file to first Cassandra instance..."
)
# Upload keyspace definitions file
scp_command = 'scp %s -r %s root@%s:/usr/local/apache-cassandra/conf/keyspaces.txt' % \
(xstr(ssh_options), temp_keyspace_file, instances[0].public_dns_name)
subprocess.call(scp_command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
# remove temporary file
os.unlink(temp_keyspace_file)
def hack_config_for_multi_region(self, ssh_options, seeds):
instances = self.get_instances()
downloaded_file = "cassandra.yaml.downloaded"
for instance in instances:
# download config file
print "downloading config from %s" % instance.public_dns_name
scp_command = 'scp %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml %s' % (xstr(ssh_options), instance.public_dns_name, downloaded_file)
subprocess.call(scp_command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
print "modifying config from %s" % instance.public_dns_name
yaml = parse_yaml(urllib.urlopen(downloaded_file))
yaml['seed_provider'][0]['parameters'][0]['seeds'] = seeds
yaml['listen_address'] = str(instance.public_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
yaml['broadcast_address'] = socket.gethostbyname(str(instance.public_dns_name))
yaml['endpoint_snitch'] = 'org.apache.cassandra.locator.Ec2MultiRegionSnitch'
print "saving config from %s" % instance.public_dns_name
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_', text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
#upload config file
print "uploading new config to %s" % instance.public_dns_name
scp_command = 'scp %s %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml' % (xstr(ssh_options), temp_file, instance.public_dns_name)
subprocess.check_call(scp_command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
os.unlink(temp_file)
os.unlink(downloaded_file)
def _get_transfer_command(self, instance, file_name, ssh_options):
transfer_command = "scp %s %s root@%s:" % (xstr(ssh_options), file_name, instance.public_dns_name)
# transfer_command = self._get_standard_ssh_command(instance, ssh_options, "cat > %s" % file_name) + " < %s" % file_name
self.logger.debug("Transfer command: %s" % transfer_command)
return transfer_command
def _get_instance_status(self, role, instance):
return (role, instance.id,
instance.image_id,
instance.dns_name, instance.private_dns_name,
instance.state, xstr(instance.key_name), instance.instance_type,
str(instance.launch_time), instance.placement)
def _print_instance(self, role, instance):
print "\t".join((role, instance.id,
instance.image_id,
instance.dns_name, instance.private_dns_name,
instance.state, xstr(instance.key_name), instance.instance_type,
str(instance.launch_time), instance.placement))
def _get_transfer_command(self, instance, file_name, ssh_options):
transfer_command = "scp %s %s root@%s:" % (
xstr(ssh_options), file_name, instance.public_dns_name)
# transfer_command = self._get_standard_ssh_command(instance, ssh_options, "cat > %s" % file_name) + " < %s" % file_name
self.logger.debug("Transfer command: %s" % transfer_command)
return transfer_command
def _get_instance_status(self, role, instance):
return (role, instance.id,
instance.image_id,
instance.dns_name, instance.private_dns_name,
instance.state, xstr(instance.key_name), instance.instance_type,
str(instance.launch_time), instance.placement)
def _print_instance(self, role, instance):
print "\t".join((role, instance.id,
instance.image_id,
instance.dns_name, instance.private_dns_name,
instance.state, xstr(instance.key_name), instance.instance_type,
str(instance.launch_time), instance.placement))
