def main(argv): with Cons.MTnnl("Checking:"): checks = [] for r in Ec2Region.All(): checks.append(Check(r)) threads = [] for c in checks: t = threading.Thread(target=c.Run) threads.append(t) t.start() for t in threads: t.join() print "" for c in checks: Cons.P("%-14s %2d" % (c.region, c.max_inst))
def _WaitForServerNodes(): # Wait for all the server nodes to be up server_num_nodes_expected = int( Ec2InitUtil.GetParam(["server", "num_nodes"])) with Cons.MTnnl("Waiting for %d server node(s) with job_id %s" % (server_num_nodes_expected, Ec2InitUtil.GetJobId())): global _nm_ip _nm_ip = None while True: _nm_ip = {} r = BotoClient.Get(Ec2InitUtil.GetRegion()).describe_instances( Filters=[ { "Name": "tag:job_id", "Values": [Ec2InitUtil.GetJobId()] }, ], ) for r0 in r["Reservations"]: for i in r0["Instances"]: #Cons.P(pprint.pformat(i)) pub_ip = i["PublicIpAddress"] for t in i["Tags"]: if t["Key"] == "name": name = t["Value"] if name.startswith("s"): _nm_ip[name] = pub_ip #Cons.P(pprint.pformat(_nm_ip)) if len(_nm_ip) == server_num_nodes_expected: break # Log progress. By now, the log file is in the local EBS. sys.stdout.write(".") sys.stdout.flush() time.sleep(1) sys.stdout.write(" all up.\n") Util.RunSubp("mkdir -p /mnt/local-ssd0/mutant/.run", ) Util.RunSubp("rm /home/ubuntu/work/mutant/.run || true") Util.RunSubp( "ln -s /mnt/local-ssd0/mutant/.run /home/ubuntu/work/mutant/.run") fn = "/home/ubuntu/work/mutant/.run/cassandra-server-ips" with open(fn, "w") as fo: fo.write(" ".join(v for k, v in _nm_ip.iteritems())) Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
def _WaitForCassServers(): with Cons.MTnnl("Wating for %d Cassandra server(s) " % len(_nm_ip)): # Query the first server node in the dict server_ip = _nm_ip.itervalues().next() # Can you just use nodetool? Yes, but needs additional authentication step # to prevent anyone from checking on that. cqlsh is already open to # everyone. Better keep the number of open services minimal. # - http://stackoverflow.com/questions/15299302/cassandra-nodetool-connection-timed-out while True: cqlsh = "%s/work/mutant/cassandra/bin/cqlsh" % os.path.expanduser( "~") lines = Util.RunSubp("%s -e \"select count(*) from system.peers\" %s || true" \ % (cqlsh, server_ip), print_cmd=False, print_output=False) if lines.startswith("Connection error:"): time.sleep(1) sys.stdout.write(".") sys.stdout.flush() continue # count # ------- # 0 # (1 rows) # Warnings : # Aggregation query used without partition key m = re.match(r"\n\s+count\n-+\n\s*(?P<count>\d+)\n.*", lines) if m is None: raise RuntimeError("Unexpected [%s]" % lines) if len(_nm_ip) == (1 + int(m.group("count"))): break time.sleep(1) sys.stdout.write(".") sys.stdout.flush() continue sys.stdout.write("all up\n")
def EditCassConf(): fn_cass_yaml = "/home/ubuntu/work/mutant/cassandra/conf/cassandra.yaml" with Cons.MT("Editing %s ..." % fn_cass_yaml): # Wait for all the server nodes to be up server_num_nodes_expected = int( Ec2InitUtil.GetParam(["server", "num_nodes"])) with Cons.MTnnl("Waiting for %d server node(s) with job_id %s" % (server_num_nodes_expected, Ec2InitUtil.GetJobId())): global _nm_ip _nm_ip = None while True: _nm_ip = {} r = BotoClient.Get(Ec2InitUtil.GetRegion()).describe_instances( Filters=[ { "Name": "tag:job_id", "Values": [Ec2InitUtil.GetJobId()] }, ], ) for r0 in r["Reservations"]: for i in r0["Instances"]: #Cons.P(pprint.pformat(i)) pub_ip = i["PublicIpAddress"] for t in i["Tags"]: if t["Key"] == "name": name = t["Value"] if name.startswith("s"): _nm_ip[name] = pub_ip #Cons.P(pprint.pformat(_nm_ip)) if len(_nm_ip) == server_num_nodes_expected: break # Log progress. By now, the log file is in the local EBS. sys.stdout.write(".") sys.stdout.flush() time.sleep(1) sys.stdout.write(" all up.\n") # Update cassandra cluster name if specified. No need to. #if "cass_cluster_name" in _tags: # # http://stackoverflow.com/questions/7517632/how-do-i-escape-double-and-single-quotes-in-sed-bash # Util.RunSubp("sed -i 's/^cluster_name: .*/cluster_name: '\"'\"'%s'\"'\"'/g' %s" # % (_tags["cass_cluster_name"], fn_cass_yaml)) Util.RunSubp("sed -i 's/" \ "^ - seeds: .*" \ "/ - seeds: \"%s\"" \ "/g' %s" % (",".join(v for k, v in _nm_ip.items()), fn_cass_yaml)) Util.RunSubp("sed -i 's/" \ "^listen_address: localhost" \ "/#listen_address: localhost" \ "/g' %s" % fn_cass_yaml) Util.RunSubp("sed -i 's/" \ "^# listen_interface: eth0" \ "/listen_interface: eth0" \ "/g' %s" % fn_cass_yaml) # sed doesn't support "?" # http://stackoverflow.com/questions/4348166/using-with-sed Util.RunSubp("sed -i 's/" \ "^\(# \|\)broadcast_address: .*" \ "/broadcast_address: %s" \ "/g' %s" % (Ec2InitUtil.GetPubIp(), fn_cass_yaml)) Util.RunSubp("sed -i 's/" \ "^rpc_address: localhost" \ "/#rpc_address: localhost" \ "/g' %s" % fn_cass_yaml) Util.RunSubp("sed -i 's/" \ "^# rpc_interface: eth1" \ "/rpc_interface: eth0" \ "/g' %s" % fn_cass_yaml) Util.RunSubp("sed -i 's/" \ "^\(# \|\)broadcast_rpc_address: .*" \ "/broadcast_rpc_address: %s" \ "/g' %s" % (Ec2InitUtil.GetPubIp(), fn_cass_yaml)) Util.RunSubp("sed -i 's/" \ "^\(#\|\)concurrent_compactors: .*" \ "/concurrent_compactors: %d" \ "/g' %s" % (multiprocessing.cpu_count(), fn_cass_yaml)) Util.RunSubp("sed -i 's/" \ "^\(#\|\)memtable_flush_writers: .*" \ "/memtable_flush_writers: %d" \ "/g' %s" % (multiprocessing.cpu_count(), fn_cass_yaml)) _EditCassConfDataFileDir(fn_cass_yaml)