def launch(self):
SIGINTWatcher(self.cleanup_after_kill)
t_start = time.time()
ami = self.config.get_ami()
keypair = self.config.get_keypair()
insttypes = self.config.get_instance_type()
zone = self.config.get_ec2_zone()
# Parse the instance type option
role_insttype = {}
for ri in insttypes.split():
role, insttype = ri.split(":")
role_insttype[role] = insttype
default_insttype = role_insttype["*"]
log.init_logging(self.loglevel)
try:
log.debug("Connecting to EC2...")
self.conn = create_ec2_connection()
if self.conn == None:
print "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables are not set."
exit(1)
log.debug("Connected to EC2.")
except BotoClientError, exc:
print "\033[1;31mERROR\033[0m - Could not connect to EC2."
print " Reason: %s" % exc.reason
exit(1)
def thread_success(self, thread):
with self.lock:
self.done_threads += 1
log.debug("%s thread has finished successfully." % thread.name)
log.debug("%i threads are done. Remaining: %s" % (self.done_threads, ",".join([t.name for t in self.threads.values() if t.status == -1])))
for t in [th for th in self.threads.values() if th.depends == thread]:
t.start()
if self.done_threads == self.num_threads:
self.all_done.set()
def scp_dir(self, fromdir, todir):
for root, dirs, files in walk(fromdir):
todir_full = todir + "/" + root[len(fromdir):]
try:
self.sftp.stat(todir_full)
except IOError, e:
self.sftp.mkdir(todir_full)
for f in files:
fromfile = root + "/" + f
tofile = todir_full + "/" + f
self.sftp.put(fromfile, tofile)
log.debug("scp %s -> %s:%s" % (fromfile, self.hostname, tofile))
def thread_failure(self, thread):
with self.lock:
if not isinstance(thread.exception, ThreadAbortException):
log.debug("%s thread has failed." % thread.name)
self.abort.set()
else:
log.debug("%s thread is being aborted." % thread.name)
thread.status = 2
self.done_threads += 1
log.debug("%i threads are done. Remaining: %s" % (self.done_threads, ",".join([t.name for t in self.threads.values() if t.status == -1])))
if self.done_threads == self.num_threads:
self.all_done.set()
def run(self, command, outf=None, errf=None, exception_on_error = True, expectnooutput=False):
channel = self.client.get_transport().open_session()
log.debug("%s - Running %s" % (self.hostname,command))
if expectnooutput:
outf = None
errf = None
else:
if outf != None:
outf = open(outf, "w")
else:
outf = self.default_outf
if errf != None:
errf = open(errf, "w")
else:
errf = self.default_errf
try:
channel.exec_command(command)
if outf != None or errf != None:
while True:
rl, wl, xl = select.select([channel],[],[])
if len(rl) > 0:
# Must be stdout
x = channel.recv(1)
if not x: break
outf.write(x)
outf.flush()
if outf != sys.stdout:
outf.close()
if errf != sys.stderr:
outf.close()
log.debug("%s - Waiting for exit status: %s" % (self.hostname,command))
rc = channel.recv_exit_status()
log.debug("%s - Ran %s" % (self.hostname,command))
channel.close()
except Exception, e:
raise # Replace by something more meaningful
def run2(self):
node = self.node
instance = self.instance
log.info("Setting up instance %s. Hostname: %s" % (instance.id, instance.public_dns_name), node)
if self.launcher.config.has_snap():
log.debug("Creating Chef volume.", node)
vol = self.launcher.conn.create_volume(1, instance.placement, self.launcher.config.get_snap())
log.debug("Created Chef volume %s. Attaching." % vol.id, node)
vol.attach(instance.id, '/dev/sdh')
log.debug("Chef volume attached. Waiting for it to become in-use.", node)
self.launcher.wait_state(vol, "in-use")
log.debug("Volume is in-use", node)
self.launcher.vols.append(vol)
self.check_continue()
if self.launcher.loglevel in (0,1):
ssh_out = None
ssh_err = None
elif self.launcher.loglevel == 2:
ssh_out = sys.stdout
ssh_err = sys.stderr
log.debug("Establishing SSH connection", node)
ssh = SSH("ubuntu", instance.public_dns_name, self.launcher.config.get_keyfile(), ssh_out, ssh_err)
ssh.open()
log.debug("SSH connection established", node)
self.check_continue()
if self.launcher.config.has_snap():
log.debug("Mounting Chef volume", node)
ssh.run("sudo mount -t ext3 /dev/sdh /chef", expectnooutput=True)
# Upload host file and update hostname
log.debug("Uploading host file and updating hostname", node)
ssh.scp("%s/hosts_ec2" % self.launcher.generated_dir,
"/chef/cookbooks/demogrid/files/default/hosts")
ssh.run("sudo cp /chef/cookbooks/demogrid/files/default/hosts /etc/hosts", expectnooutput=True)
ssh.run("sudo bash -c \"echo %s > /etc/hostname\"" % node.hostname, expectnooutput=True)
ssh.run("sudo /etc/init.d/hostname restart")
self.check_continue()
# Upload topology file
log.debug("Uploading topology file", node)
ssh.scp("%s/topology_ec2.rb" % self.launcher.generated_dir,
"/chef/cookbooks/demogrid/attributes/topology.rb")
# Copy certificates
log.debug("Copying certificates", node)
ssh.scp_dir("%s/certs" % self.launcher.generated_dir,
"/chef/cookbooks/demogrid/files/default/")
self.check_continue()
if self.launcher.loglevel == 0:
print " \033[1;37m%s\033[0m: Basic setup is done. Installing Grid software now." % node.hostname.split(".")[0]
# Run chef
log.debug("Running chef", node)
ssh.scp("%s/lib/ec2/chef.conf" % self.launcher.demogrid_dir,
"/tmp/chef.conf")
ssh.run("echo '{ \"run_list\": \"role[%s]\" }' > /tmp/chef.json" % node.role, expectnooutput=True)
ssh.run("sudo chef-solo -c /tmp/chef.conf -j /tmp/chef.json")
self.check_continue()
# The Chef recipes will overwrite the hostname, so
# we need to set it again.
ssh.run("sudo bash -c \"echo %s > /etc/hostname\"" % node.hostname, expectnooutput=True)
ssh.run("sudo /etc/init.d/hostname restart")
if self.launcher.config.has_snap():
ssh.run("sudo umount /chef", expectnooutput=True)
vol.detach()
self.launcher.wait_state(vol, "available")
vol.delete()
self.launcher.vols.remove(vol)
ssh.run("sudo update-rc.d nis enable")
log.info("Configuration done.", node)
if self.launcher.loglevel == 0:
print " \033[1;37m%s\033[0m is ready." % node.hostname.split(".")[0]
log.debug("Connecting to EC2...")
self.conn = create_ec2_connection()
if self.conn == None:
print "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables are not set."
exit(1)
log.debug("Connected to EC2.")
except BotoClientError, exc:
print "\033[1;31mERROR\033[0m - Could not connect to EC2."
print " Reason: %s" % exc.reason
exit(1)
except Exception, exc:
self.handle_unexpected_exception(exc)
# Load topology
log.debug("Loading topology file...")
f = open ("%s/topology.dat" % self.generated_dir, "r")
topology = load(f)
f.close()
log.debug("Loaded topology file")
if self.config.get_ec2_access_type() == "public":
topology.global_attributes["ec2_public"] = "true"
else:
topology.global_attributes["ec2_public"] = "false"
nodes = topology.get_nodes()
num_instances = len(nodes)
nodes_by_role = {}
for n in nodes:
else:
return rc
def scp(self, fromf, tof):
try:
self.sftp.put(fromf, tof)
except Exception, e:
traceback.print_exc()
try:
self.close()
except:
pass
log.debug("scp %s -> %s:%s" % (fromf, self.hostname, tof))
def scp_dir(self, fromdir, todir):
for root, dirs, files in walk(fromdir):
todir_full = todir + "/" + root[len(fromdir):]
try:
self.sftp.stat(todir_full)
except IOError, e:
self.sftp.mkdir(todir_full)
for f in files:
fromfile = root + "/" + f
tofile = todir_full + "/" + f
self.sftp.put(fromfile, tofile)
log.debug("scp %s -> %s:%s" % (fromfile, self.hostname, tofile))