def hack_config_for_multi_region(self, ssh_options, seeds):
instances = self.get_instances()
downloaded_file = "cassandra.yaml.downloaded"
for instance in instances:
# download config file
print "downloading config from %s" % instance.public_dns_name
scp_command = 'scp %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml %s' % (xstr(ssh_options), instance.public_dns_name, downloaded_file)
subprocess.call(scp_command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
print "modifying config from %s" % instance.public_dns_name
yaml = parse_yaml(urllib.urlopen(downloaded_file))
yaml['seed_provider'][0]['parameters'][0]['seeds'] = seeds
yaml['listen_address'] = str(instance.public_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
yaml['broadcast_address'] = socket.gethostbyname(str(instance.public_dns_name))
yaml['endpoint_snitch'] = 'org.apache.cassandra.locator.Ec2MultiRegionSnitch'
print "saving config from %s" % instance.public_dns_name
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_', text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
#upload config file
print "uploading new config to %s" % instance.public_dns_name
scp_command = 'scp %s %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml' % (xstr(ssh_options), temp_file, instance.public_dns_name)
subprocess.check_call(scp_command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
os.unlink(temp_file)
os.unlink(downloaded_file)
def _modify_config_file(self,
instance,
config_file,
seed_ips,
token,
set_tokens=True,
auto_bootstrap=False):
# YAML (0.7.x+)
if config_file.endswith(".yaml"):
remote_file = "cassandra.yaml"
yaml = parse_yaml(urllib.urlopen(config_file))
yaml['seed_provider'][0]['parameters'][0]['seeds'] = ",".join(
seed_ips)
if set_tokens is True:
yaml['initial_token'] = token
if auto_bootstrap:
yaml['auto_bootstrap'] = 'true'
yaml['data_file_directories'] = ['/mnt/cassandra-data']
yaml['commitlog_directory'] = '/mnt/cassandra-logs'
yaml['listen_address'] = str(instance.public_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_',
text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
else:
raise Exception(
"Configuration file must be yaml (implies Cassandra 0.7.x or greater)"
)
return temp_file, remote_file
def hack_config_for_multi_region(self, ssh_options, seeds):
instances = self.get_instances()
downloaded_file = os.path.join("/tmp", "cassandra.yaml.downloaded")
for instance in instances:
with settings(host_string=instance.public_dns_name, warn_only=True):
# download config file
print "downloading config from %s" % instance.public_dns_name
get("/etc/cassandra/cassandra.yaml", downloaded_file)
print "modifying config from %s" % instance.public_dns_name
yaml = parse_yaml(urllib.urlopen(downloaded_file))
yaml['seed_provider'][0]['parameters'][0]['seeds'] = seeds
yaml['listen_address'] = str(instance.public_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
yaml['broadcast_address'] = socket.gethostbyname(str(instance.public_dns_name))
yaml['endpoint_snitch'] = 'org.apache.cassandra.locator.Ec2MultiRegionSnitch'
print "saving config from %s" % instance.public_dns_name
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_', text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
#upload config file
print "uploading new config to %s" % instance.public_dns_name
put(temp_file, "/etc/cassandra/cassandra.yaml", use_sudo=use_sudo())
os.unlink(temp_file)
os.unlink(downloaded_file)
def _get_config_value(self, config_file, yaml_name, xml_name):
if config_file.endswith(".xml") :
xml = parse_xml(urllib.urlopen(config_file)).getroot()
return xml.find(xml_name).text
elif config_file.endswith(".yaml") :
yaml = parse_yaml(urllib.urlopen(config_file))
return yaml[yaml_name]
else:
raise Exception("Configuration file must be on of xml or yaml")
def _get_config_value(self, config_file, yaml_name, xml_name):
if config_file.endswith(".xml"):
xml = parse_xml(urllib.urlopen(config_file)).getroot()
return xml.find(xml_name).text
elif config_file.endswith(".yaml"):
yaml = parse_yaml(urllib.urlopen(config_file))
return yaml[yaml_name]
else:
raise Exception("Configuration file must be on of xml or yaml")
def _configure_cassandra_instance(self, instance, seed_ips, token, set_tokens=True, auto_bootstrap=False):
self.logger.debug("Configuring %s..." % instance.id)
yaml_file = os.path.join("/tmp", "cassandra.yaml")
cassandra_home = self.get_cassandra_home(instance)
self.logger.debug("Local cassandra.yaml file: %s" % yaml_file)
with settings(host_string=instance.public_dns_name, warn_only=True): #, hide("everything"):
cassandra_data = os.path.join("/mnt", "cassandra-data")
cassandra_logs = os.path.join("/mnt", "cassandra-logs")
# create directories and log files
exec_command("mkdir -p %s" % cassandra_data)
exec_command("mkdir -p %s" % cassandra_logs)
# set permissions
exec_command("chown -R cassandra:cassandra %s %s" % (cassandra_data, cassandra_logs))
try:
# get yaml file
get(os.path.join(cassandra_home, "conf", "cassandra.yaml"), "/tmp")
# modify it
f = open(yaml_file)
yaml = parse_yaml(f)
f.close()
yaml['seed_provider'][0]['parameters'][0]['seeds'] = ",".join(seed_ips)
if set_tokens is True :
yaml['initial_token'] = token
if auto_bootstrap :
yaml['auto_bootstrap'] = 'true'
yaml['data_file_directories'] = [cassandra_data]
yaml['commitlog_directory'] = cassandra_logs
yaml['listen_address'] = str(instance.private_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
f = open(yaml_file, "w")
f.write(dump_yaml(yaml))
f.close()
# put modified yaml file
put(yaml_file, os.path.join(cassandra_home, "conf", "cassandra.yaml"), use_sudo=use_sudo())
except SystemExit, e:
raise
pass
def hack_config_for_multi_region(self, ssh_options, seeds):
instances = self.get_instances()
downloaded_file = "cassandra.yaml.downloaded"
for instance in instances:
# download config file
print "downloading config from %s" % instance.public_dns_name
scp_command = 'scp %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml %s' % (
xstr(ssh_options), instance.public_dns_name, downloaded_file)
subprocess.call(scp_command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
print "modifying config from %s" % instance.public_dns_name
yaml = parse_yaml(urllib.urlopen(downloaded_file))
yaml['seed_provider'][0]['parameters'][0]['seeds'] = seeds
yaml['listen_address'] = str(instance.public_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
yaml['broadcast_address'] = socket.gethostbyname(
str(instance.public_dns_name))
yaml[
'endpoint_snitch'] = 'org.apache.cassandra.locator.Ec2MultiRegionSnitch'
print "saving config from %s" % instance.public_dns_name
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_',
text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
#upload config file
print "uploading new config to %s" % instance.public_dns_name
scp_command = 'scp %s %s root@%s:/usr/local/apache-cassandra/conf/cassandra.yaml' % (
xstr(ssh_options), temp_file, instance.public_dns_name)
subprocess.check_call(scp_command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
os.unlink(temp_file)
os.unlink(downloaded_file)
def _modify_config_file(self, instance, config_file, seed_ips, token, set_tokens=True, auto_bootstrap=False):
# YAML (0.7.x+)
if config_file.endswith(".yaml"):
remote_file = "cassandra.yaml"
yaml = parse_yaml(urllib.urlopen(config_file))
yaml['seed_provider'][0]['parameters'][0]['seeds'] = ",".join(seed_ips)
if set_tokens is True :
yaml['initial_token'] = token
if auto_bootstrap :
yaml['auto_bootstrap'] = 'true'
yaml['data_file_directories'] = ['/mnt/cassandra-data']
yaml['commitlog_directory'] = '/mnt/cassandra-logs'
yaml['listen_address'] = str(instance.public_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_', text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
else:
raise Exception("Configuration file must be yaml (implies Cassandra 0.7.x or greater)")
return temp_file, remote_file
def _modify_config_file(self, instance, config_file, seed_ips, token):
# XML (0.6.x)
if config_file.endswith(".xml"):
remote_file = "storage-conf.xml"
xml = parse_xml(urllib.urlopen(config_file)).getroot()
# Seeds
seeds = xml.find("Seeds")
if seeds is not None:
while seeds.getchildren():
seeds.remove(seeds.getchildren()[0])
else:
seeds = Element("Seeds")
xml.append(seeds)
for seed_ip in seed_ips:
seed = Element("Seed")
seed.text = seed_ip
seeds.append(seed)
# Initial token
initial_token = xml.find("InitialToken")
if initial_token is None:
initial_token = Element("InitialToken")
xml.append(initial_token)
initial_token.text = token
# Logs
commit_log_directory = xml.find("CommitLogDirectory")
if commit_log_directory is None:
commit_log_directory = Element("CommitLogDirectory")
xml.append(commit_log_directory)
commit_log_directory.text = "/mnt/cassandra-logs"
# Data
data_file_directories = xml.find("DataFileDirectories")
if data_file_directories is not None:
while data_file_directories.getchildren():
data_file_directories.remove(data_file_directories.getchildren()[0])
else:
data_file_directories = Element("DataFileDirectories")
xml.append(data_file_directories)
data_file_directory = Element("DataFileDirectory")
data_file_directory.text = "/mnt/cassandra-data"
data_file_directories.append(data_file_directory)
# listen address
listen_address = xml.find("ListenAddress")
if listen_address is None:
listen_address = Element("ListenAddress")
xml.append(listen_address)
listen_address.text = ""
# thrift address
thrift_address = xml.find("ThriftAddress")
if thrift_address is None:
thrift_address = Element("ThriftAddress")
xml.append(thrift_address)
thrift_address.text = ""
fd, temp_file = tempfile.mkstemp(prefix='storage-conf.xml_', text=True)
os.write(fd, dump_xml(xml))
os.close(fd)
# YAML (0.7.x)
elif config_file.endswith(".yaml"):
remote_file = "cassandra.yaml"
yaml = parse_yaml(urllib.urlopen(config_file))
yaml['seeds'] = seed_ips
yaml['initial_token'] = token
yaml['data_file_directories'] = ['/mnt/cassandra-data']
yaml['commitlog_directory'] = '/mnt/cassandra-logs'
yaml['listen_address'] = str(instance.private_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_', text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
else:
raise Exception("Configuration file must be one of xml or yaml")
return temp_file, remote_file
def _modify_config_file(self, instance, config_file, seed_ips, token):
# XML (0.6.x)
if config_file.endswith(".xml"):
remote_file = "storage-conf.xml"
xml = parse_xml(urllib.urlopen(config_file)).getroot()
# Seeds
seeds = xml.find("Seeds")
if seeds is not None:
while seeds.getchildren():
seeds.remove(seeds.getchildren()[0])
else:
seeds = Element("Seeds")
xml.append(seeds)
for seed_ip in seed_ips:
seed = Element("Seed")
seed.text = seed_ip
seeds.append(seed)
# Initial token
initial_token = xml.find("InitialToken")
if initial_token is None:
initial_token = Element("InitialToken")
xml.append(initial_token)
initial_token.text = token
# Logs
commit_log_directory = xml.find("CommitLogDirectory")
if commit_log_directory is None:
commit_log_directory = Element("CommitLogDirectory")
xml.append(commit_log_directory)
commit_log_directory.text = "/mnt/cassandra-logs"
# Data
data_file_directories = xml.find("DataFileDirectories")
if data_file_directories is not None:
while data_file_directories.getchildren():
data_file_directories.remove(
data_file_directories.getchildren()[0])
else:
data_file_directories = Element("DataFileDirectories")
xml.append(data_file_directories)
data_file_directory = Element("DataFileDirectory")
data_file_directory.text = "/mnt/cassandra-data"
data_file_directories.append(data_file_directory)
# listen address
listen_address = xml.find("ListenAddress")
if listen_address is None:
listen_address = Element("ListenAddress")
xml.append(listen_address)
listen_address.text = ""
# thrift address
thrift_address = xml.find("ThriftAddress")
if thrift_address is None:
thrift_address = Element("ThriftAddress")
xml.append(thrift_address)
thrift_address.text = ""
fd, temp_file = tempfile.mkstemp(prefix='storage-conf.xml_',
text=True)
os.write(fd, dump_xml(xml))
os.close(fd)
# YAML (0.7.x)
elif config_file.endswith(".yaml"):
remote_file = "cassandra.yaml"
yaml = parse_yaml(urllib.urlopen(config_file))
yaml['seeds'] = seed_ips
yaml['initial_token'] = token
yaml['data_file_directories'] = ['/mnt/cassandra-data']
yaml['commitlog_directory'] = '/mnt/cassandra-logs'
yaml['listen_address'] = str(instance.private_dns_name)
yaml['rpc_address'] = str(instance.public_dns_name)
fd, temp_file = tempfile.mkstemp(prefix='cassandra.yaml_',
text=True)
os.write(fd, dump_yaml(yaml))
os.close(fd)
else:
raise Exception("Configuration file must be one of xml or yaml")
return temp_file, remote_file
def get_meta(filestring):
empty, meta, post = filestring.split('---')
meta_dict = parse_yaml(meta)
return meta_dict
def parse(self, source):
return parse_yaml(source)
def get_meta(filestring):
empty, meta, post = filestring.split('---')
meta_dict = parse_yaml(meta)
return meta_dict
def get_config(config_dir, config_name):
return parse_yaml(open(join(config_dir, config_name)))
