def multiremote_from_cluster_stdout(self, stdout, mname='ambari'): """ take the output from the up_aws_cluster script and parse it first check it was successful, then create a multiremote object to contact all machines Return multiremote, [iids] """ import kavedeploy as lD connectcmd = "" adict = stdout.split("\n")[-2].replace("Complete, created:", "") # try interpreting as json adict = d2j(adict) iids = [deets[0] for _name, deets in adict.iteritems()] ips = [deets[1] for _name, deets in adict.iteritems()] testm = adict.keys()[0] connectcmd = "" for line in range(len(stdout.split('\n'))): if (testm + " connect remotely with") in stdout.split("\n")[line]: connectcmd = stdout.split("\n")[line + 1].strip() iid, ip = adict[testm] self.assertTrue(ip in connectcmd, ip + " Wrong IP seen for connecting to " + testm + ' ' + connectcmd) jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"])) import json acconf = json.loads(jsondat.read()) jsondat.close() keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"] self.assertTrue(keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd, "wrong keyfile seen in (" + connectcmd + ")") return lD.multiremotes(["ssh:root@" + str(ip) for ip in ips], access_key=keyfile), iids
def multiremote_from_cluster_stdout(self, stdout, mname='ambari'): """ take the output from the up_aws_cluster script and parse it first check it was successful, then create a multiremote object to contact all machines Return multiremote, [iids] """ import kavedeploy as lD connectcmd = "" adict = stdout.split("\n")[-2].replace("Complete, created:", "") # try interpreting as json adict = d2j(adict) iids = [deets[0] for _name, deets in adict.iteritems()] ips = [deets[1] for _name, deets in adict.iteritems()] testm = adict.keys()[0] connectcmd = "" for line in range(len(stdout.split('\n'))): if (testm + " connect remotely with") in stdout.split("\n")[line]: connectcmd = stdout.split("\n")[line + 1].strip() iid, ip = adict[testm] self.assertTrue( ip in connectcmd, ip + " Wrong IP seen for connecting to " + testm + ' ' + connectcmd) jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"])) import json acconf = json.loads(jsondat.read()) jsondat.close() keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"] self.assertTrue( keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd, "wrong keyfile seen in (" + connectcmd + ")") return lD.multiremotes(["ssh:root@" + str(ip) for ip in ips], access_key=keyfile), iids
def runTest(self): """ Tests which cehck the function of the deployment library, but do not need any environment parameters or access to aws """ import kavedeploy as lD lD.testproxy() self.assertIsNot(lD.which("ls"), None) self.assertRaises(RuntimeError, lD.run_quiet, ("thisisnotacommand")) stdout = lD.run_quiet(['which', 'ls'], shell=False) self.assertTrue('/bin/ls' in stdout) self.assertIsNot(lD.which("pdsh"), None, "pdsh is not installed, please install it in order to test the multiremotes functionality, " "sudo yum -y install pdsh") lD.run_quiet("touch /tmp/fake_test_ssh_key.pem") lD.run_quiet("chmod 400 /tmp/fake_test_ssh_key.pem") test = lD.remoteHost("root", "test", '/tmp/fake_test_ssh_key.pem') test = lD.multiremotes([test.host], access_key='/tmp/fake_test_ssh_key.pem')
# Start ambari agents ################################################################## print "Attempting to start ambari agents on all", len(hosts), "nodes" sys.stdout.flush() ambari = lD.remoteHost("root", thehost, access_key) # Step one, install myself, dsh and deploy ambari agents to all nodes lD.install_pdsh(ambari) # modify iptables, only in case of Centos6 lD.disable_security(ambari) admin = ambari.run("hostname") whole_cluster = lD.multiremotes(hosts, jump=ambari) # Check if all nodes in the cluster are contactable try: whole_cluster.check(firsttime=True) except lD.ShellExecuteError: print "Could not access machines with passwordless ssh, the ambari node must have passwordless ssh access to the " \ "rest, fix and try again" raise # Verify that all nodes have similar system times ambtime = int(ambari.run("date -u '+%s'")) cltime = whole_cluster.run("date -u '+%s'") cltime = [(int(p.split(':')[-1])) for p in cltime.split('\n')] if (max(cltime) - min(cltime)) > (10 * 60):
remoteuser = lA.default_usernamedict[lA.default_os] remote = lD.remoteHost(remoteuser, ip, amazon_keyfile) lD.wait_until_up(remote, 20) remote = lD.remote_cp_authkeys(remote, 'root') if "Tags" in security_config: resources = lA.find_all_child_resources(iid) lA.tag_resources(resources, security_config["Tags"]) remote.register() instance_to_remote[iid] = remote allremotes = [ "ssh:root@" + remote.host for remote in instance_to_remote.values() ] allremotes = lD.multiremotes(list_of_hosts=allremotes, access_key=amazon_keyfile) print "test local PDSH, install pdcp" print allremotes.run("echo yes") allremotes.run("yum clean all") lD.install_pdsh(allremotes) print "====================================" print "configure SSH on all machines" print "====================================" lD.confallssh(allremotes, restart=False) lD.confsshpermissions(allremotes) print "====================================" print "name the instances" print "====================================" sys.stdout.flush()
if ip is None: raise SystemError(iid + " no ip assigned after quite some time") remoteuser = lA.default_usernamedict[lA.default_os] remote = lD.remoteHost(remoteuser, ip, amazon_keyfile) lD.wait_until_up(remote, 20) remote = lD.remote_cp_authkeys(remote, 'root') if "Tags" in security_config: resources = lA.find_all_child_resources(iid) lA.tag_resources(resources, security_config["Tags"]) remote.register() instance_to_remote[iid] = remote allremotes = ["ssh:root@" + remote.host for remote in instance_to_remote.values()] allremotes = lD.multiremotes(list_of_hosts=allremotes, access_key=amazon_keyfile) print "test local PDSH, install pdcp" print allremotes.run("echo yes") allremotes.run("yum clean all") lD.install_pdsh(allremotes) print "====================================" print "configure SSH on all machines" print "====================================" lD.confallssh(allremotes, restart=False) lD.confsshpermissions(allremotes) print "====================================" print "name the instances" print "====================================" sys.stdout.flush()