def multiremote_from_cluster_stdout(self, stdout, mname='ambari'):
"""
take the output from the up_aws_cluster script and parse it
first check it was successful, then create a multiremote object
to contact all machines
Return multiremote, [iids]
"""
import kavedeploy as lD
connectcmd = ""
adict = stdout.split("\n")[-2].replace("Complete, created:", "")
# try interpreting as json
adict = d2j(adict)
iids = [deets[0] for _name, deets in adict.iteritems()]
ips = [deets[1] for _name, deets in adict.iteritems()]
testm = adict.keys()[0]
connectcmd = ""
for line in range(len(stdout.split('\n'))):
if (testm + " connect remotely with") in stdout.split("\n")[line]:
connectcmd = stdout.split("\n")[line + 1].strip()
iid, ip = adict[testm]
self.assertTrue(ip in connectcmd, ip + " Wrong IP seen for connecting to " + testm + ' ' + connectcmd)
jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
import json
acconf = json.loads(jsondat.read())
jsondat.close()
keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
self.assertTrue(keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
"wrong keyfile seen in (" + connectcmd + ")")
return lD.multiremotes(["ssh:root@" + str(ip) for ip in ips], access_key=keyfile), iids
def multiremote_from_cluster_stdout(self, stdout, mname='ambari'):
"""
take the output from the up_aws_cluster script and parse it
first check it was successful, then create a multiremote object
to contact all machines
Return multiremote, [iids]
"""
import kavedeploy as lD
connectcmd = ""
adict = stdout.split("\n")[-2].replace("Complete, created:", "")
# try interpreting as json
adict = d2j(adict)
iids = [deets[0] for _name, deets in adict.iteritems()]
ips = [deets[1] for _name, deets in adict.iteritems()]
testm = adict.keys()[0]
connectcmd = ""
for line in range(len(stdout.split('\n'))):
if (testm + " connect remotely with") in stdout.split("\n")[line]:
connectcmd = stdout.split("\n")[line + 1].strip()
iid, ip = adict[testm]
self.assertTrue(
ip in connectcmd, ip + " Wrong IP seen for connecting to " +
testm + ' ' + connectcmd)
jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
import json
acconf = json.loads(jsondat.read())
jsondat.close()
keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
self.assertTrue(
keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
"wrong keyfile seen in (" + connectcmd + ")")
return lD.multiremotes(["ssh:root@" + str(ip) for ip in ips],
access_key=keyfile), iids
def runTest(self):
"""
Tests which cehck the function of the deployment library,
but do not need any environment parameters or access to aws
"""
import kavedeploy as lD
lD.testproxy()
self.assertIsNot(lD.which("ls"), None)
self.assertRaises(RuntimeError, lD.run_quiet, ("thisisnotacommand"))
stdout = lD.run_quiet(['which', 'ls'], shell=False)
self.assertTrue('/bin/ls' in stdout)
self.assertIsNot(lD.which("pdsh"), None,
"pdsh is not installed, please install it in order to test the multiremotes functionality, "
"sudo yum -y install pdsh")
lD.run_quiet("touch /tmp/fake_test_ssh_key.pem")
lD.run_quiet("chmod 400 /tmp/fake_test_ssh_key.pem")
test = lD.remoteHost("root", "test", '/tmp/fake_test_ssh_key.pem')
test = lD.multiremotes([test.host], access_key='/tmp/fake_test_ssh_key.pem')
# Start ambari agents
##################################################################
print "Attempting to start ambari agents on all", len(hosts), "nodes"
sys.stdout.flush()
ambari = lD.remoteHost("root", thehost, access_key)
# Step one, install myself, dsh and deploy ambari agents to all nodes
lD.install_pdsh(ambari)
# modify iptables, only in case of Centos6
lD.disable_security(ambari)
admin = ambari.run("hostname")
whole_cluster = lD.multiremotes(hosts, jump=ambari)
# Check if all nodes in the cluster are contactable
try:
whole_cluster.check(firsttime=True)
except lD.ShellExecuteError:
print "Could not access machines with passwordless ssh, the ambari node must have passwordless ssh access to the " \
"rest, fix and try again"
raise
# Verify that all nodes have similar system times
ambtime = int(ambari.run("date -u '+%s'"))
cltime = whole_cluster.run("date -u '+%s'")
cltime = [(int(p.split(':')[-1])) for p in cltime.split('\n')]
if (max(cltime) - min(cltime)) > (10 * 60):
# Start ambari agents
##################################################################
print "Attempting to start ambari agents on all", len(hosts), "nodes"
sys.stdout.flush()
ambari = lD.remoteHost("root", thehost, access_key)
# Step one, install myself, dsh and deploy ambari agents to all nodes
lD.install_pdsh(ambari)
# modify iptables, only in case of Centos6
lD.disable_security(ambari)
admin = ambari.run("hostname")
whole_cluster = lD.multiremotes(hosts, jump=ambari)
# Check if all nodes in the cluster are contactable
try:
whole_cluster.check(firsttime=True)
except lD.ShellExecuteError:
print "Could not access machines with passwordless ssh, the ambari node must have passwordless ssh access to the " \
"rest, fix and try again"
raise
# Verify that all nodes have similar system times
ambtime = int(ambari.run("date -u '+%s'"))
cltime = whole_cluster.run("date -u '+%s'")
cltime = [(int(p.split(':')[-1])) for p in cltime.split('\n')]
if (max(cltime) - min(cltime)) > (10 * 60):
remoteuser = lA.default_usernamedict[lA.default_os]
remote = lD.remoteHost(remoteuser, ip, amazon_keyfile)
lD.wait_until_up(remote, 20)
remote = lD.remote_cp_authkeys(remote, 'root')
if "Tags" in security_config:
resources = lA.find_all_child_resources(iid)
lA.tag_resources(resources, security_config["Tags"])
remote.register()
instance_to_remote[iid] = remote
allremotes = [
"ssh:root@" + remote.host for remote in instance_to_remote.values()
]
allremotes = lD.multiremotes(list_of_hosts=allremotes,
access_key=amazon_keyfile)
print "test local PDSH, install pdcp"
print allremotes.run("echo yes")
allremotes.run("yum clean all")
lD.install_pdsh(allremotes)
print "===================================="
print "configure SSH on all machines"
print "===================================="
lD.confallssh(allremotes, restart=False)
lD.confsshpermissions(allremotes)
print "===================================="
print "name the instances"
print "===================================="
sys.stdout.flush()
if ip is None:
raise SystemError(iid + " no ip assigned after quite some time")
remoteuser = lA.default_usernamedict[lA.default_os]
remote = lD.remoteHost(remoteuser, ip, amazon_keyfile)
lD.wait_until_up(remote, 20)
remote = lD.remote_cp_authkeys(remote, 'root')
if "Tags" in security_config:
resources = lA.find_all_child_resources(iid)
lA.tag_resources(resources, security_config["Tags"])
remote.register()
instance_to_remote[iid] = remote
allremotes = ["ssh:root@" + remote.host for remote in instance_to_remote.values()]
allremotes = lD.multiremotes(list_of_hosts=allremotes, access_key=amazon_keyfile)
print "test local PDSH, install pdcp"
print allremotes.run("echo yes")
allremotes.run("yum clean all")
lD.install_pdsh(allremotes)
print "===================================="
print "configure SSH on all machines"
print "===================================="
lD.confallssh(allremotes, restart=False)
lD.confsshpermissions(allremotes)
print "===================================="
print "name the instances"
print "===================================="
sys.stdout.flush()