def main(argv):
with Cons.MTnnl("Checking:"):
checks = []
for r in Ec2Region.All():
checks.append(Check(r))
threads = []
for c in checks:
t = threading.Thread(target=c.Run)
threads.append(t)
t.start()
for t in threads:
t.join()
print ""
for c in checks:
Cons.P("%-14s %2d" % (c.region, c.max_inst))
def _WaitForServerNodes():
# Wait for all the server nodes to be up
server_num_nodes_expected = int(
Ec2InitUtil.GetParam(["server", "num_nodes"]))
with Cons.MTnnl("Waiting for %d server node(s) with job_id %s" %
(server_num_nodes_expected, Ec2InitUtil.GetJobId())):
global _nm_ip
_nm_ip = None
while True:
_nm_ip = {}
r = BotoClient.Get(Ec2InitUtil.GetRegion()).describe_instances(
Filters=[
{
"Name": "tag:job_id",
"Values": [Ec2InitUtil.GetJobId()]
},
], )
for r0 in r["Reservations"]:
for i in r0["Instances"]:
#Cons.P(pprint.pformat(i))
pub_ip = i["PublicIpAddress"]
for t in i["Tags"]:
if t["Key"] == "name":
name = t["Value"]
if name.startswith("s"):
_nm_ip[name] = pub_ip
#Cons.P(pprint.pformat(_nm_ip))
if len(_nm_ip) == server_num_nodes_expected:
break
# Log progress. By now, the log file is in the local EBS.
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(1)
sys.stdout.write(" all up.\n")
Util.RunSubp("mkdir -p /mnt/local-ssd0/mutant/.run", )
Util.RunSubp("rm /home/ubuntu/work/mutant/.run || true")
Util.RunSubp(
"ln -s /mnt/local-ssd0/mutant/.run /home/ubuntu/work/mutant/.run")
fn = "/home/ubuntu/work/mutant/.run/cassandra-server-ips"
with open(fn, "w") as fo:
fo.write(" ".join(v for k, v in _nm_ip.iteritems()))
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
def _WaitForCassServers():
with Cons.MTnnl("Wating for %d Cassandra server(s) " % len(_nm_ip)):
# Query the first server node in the dict
server_ip = _nm_ip.itervalues().next()
# Can you just use nodetool? Yes, but needs additional authentication step
# to prevent anyone from checking on that. cqlsh is already open to
# everyone. Better keep the number of open services minimal.
# - http://stackoverflow.com/questions/15299302/cassandra-nodetool-connection-timed-out
while True:
cqlsh = "%s/work/mutant/cassandra/bin/cqlsh" % os.path.expanduser(
"~")
lines = Util.RunSubp("%s -e \"select count(*) from system.peers\" %s || true" \
% (cqlsh, server_ip), print_cmd=False, print_output=False)
if lines.startswith("Connection error:"):
time.sleep(1)
sys.stdout.write(".")
sys.stdout.flush()
continue
# count
# -------
# 0
# (1 rows)
# Warnings :
# Aggregation query used without partition key
m = re.match(r"\n\s+count\n-+\n\s*(?P<count>\d+)\n.*", lines)
if m is None:
raise RuntimeError("Unexpected [%s]" % lines)
if len(_nm_ip) == (1 + int(m.group("count"))):
break
time.sleep(1)
sys.stdout.write(".")
sys.stdout.flush()
continue
sys.stdout.write("all up\n")
def EditCassConf():
fn_cass_yaml = "/home/ubuntu/work/mutant/cassandra/conf/cassandra.yaml"
with Cons.MT("Editing %s ..." % fn_cass_yaml):
# Wait for all the server nodes to be up
server_num_nodes_expected = int(
Ec2InitUtil.GetParam(["server", "num_nodes"]))
with Cons.MTnnl("Waiting for %d server node(s) with job_id %s" %
(server_num_nodes_expected, Ec2InitUtil.GetJobId())):
global _nm_ip
_nm_ip = None
while True:
_nm_ip = {}
r = BotoClient.Get(Ec2InitUtil.GetRegion()).describe_instances(
Filters=[
{
"Name": "tag:job_id",
"Values": [Ec2InitUtil.GetJobId()]
},
], )
for r0 in r["Reservations"]:
for i in r0["Instances"]:
#Cons.P(pprint.pformat(i))
pub_ip = i["PublicIpAddress"]
for t in i["Tags"]:
if t["Key"] == "name":
name = t["Value"]
if name.startswith("s"):
_nm_ip[name] = pub_ip
#Cons.P(pprint.pformat(_nm_ip))
if len(_nm_ip) == server_num_nodes_expected:
break
# Log progress. By now, the log file is in the local EBS.
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(1)
sys.stdout.write(" all up.\n")
# Update cassandra cluster name if specified. No need to.
#if "cass_cluster_name" in _tags:
# # http://stackoverflow.com/questions/7517632/how-do-i-escape-double-and-single-quotes-in-sed-bash
# Util.RunSubp("sed -i 's/^cluster_name: .*/cluster_name: '\"'\"'%s'\"'\"'/g' %s"
# % (_tags["cass_cluster_name"], fn_cass_yaml))
Util.RunSubp("sed -i 's/" \
"^ - seeds: .*" \
"/ - seeds: \"%s\"" \
"/g' %s" % (",".join(v for k, v in _nm_ip.items()), fn_cass_yaml))
Util.RunSubp("sed -i 's/" \
"^listen_address: localhost" \
"/#listen_address: localhost" \
"/g' %s" % fn_cass_yaml)
Util.RunSubp("sed -i 's/" \
"^# listen_interface: eth0" \
"/listen_interface: eth0" \
"/g' %s" % fn_cass_yaml)
# sed doesn't support "?"
# http://stackoverflow.com/questions/4348166/using-with-sed
Util.RunSubp("sed -i 's/" \
"^\(# \|\)broadcast_address: .*" \
"/broadcast_address: %s" \
"/g' %s" % (Ec2InitUtil.GetPubIp(), fn_cass_yaml))
Util.RunSubp("sed -i 's/" \
"^rpc_address: localhost" \
"/#rpc_address: localhost" \
"/g' %s" % fn_cass_yaml)
Util.RunSubp("sed -i 's/" \
"^# rpc_interface: eth1" \
"/rpc_interface: eth0" \
"/g' %s" % fn_cass_yaml)
Util.RunSubp("sed -i 's/" \
"^\(# \|\)broadcast_rpc_address: .*" \
"/broadcast_rpc_address: %s" \
"/g' %s" % (Ec2InitUtil.GetPubIp(), fn_cass_yaml))
Util.RunSubp("sed -i 's/" \
"^\(#\|\)concurrent_compactors: .*" \
"/concurrent_compactors: %d" \
"/g' %s" % (multiprocessing.cpu_count(), fn_cass_yaml))
Util.RunSubp("sed -i 's/" \
"^\(#\|\)memtable_flush_writers: .*" \
"/memtable_flush_writers: %d" \
"/g' %s" % (multiprocessing.cpu_count(), fn_cass_yaml))
_EditCassConfDataFileDir(fn_cass_yaml)
