lD.mysleep(1)
ip = lA.pub_ip(iid)
acount = acount + 1
remoteuser = lA.default_usernamedict[osval]
if os.path.exists(os.path.realpath(os.path.expanduser(keyloc))):
print "waiting until contactable, ctrl-C to quit"
try:
remote = lD.remoteHost(remoteuser, ip, keyloc)
lD.wait_until_up(remote, 20)
if "Tags" in security_config:
resources = lA.find_all_child_resources(iid)
lA.tag_resources(resources, security_config["Tags"])
remote.register()
remote = lD.remote_cp_authkeys(remote, 'root')
lD.rename_remote_host(remote, machinename, 'kave.io')
lD.confallssh(remote, restart=False)
lD.confsshpermissions(remote)
if osval.startswith("Centos"):
remote.run("yum clean all")
remote.describe()
lD.disable_security(remote, firewall=False, permanent=False)
except KeyboardInterrupt:
pass
else:
print "Warning: not contactable since keyfile supplied does not exist locally,",
print "also means I could not rename the host", keyloc
print "OK, iid " + iid + " now lives at IP " + ip
print "Renaming, configuring firewall and adding more disk space"
lD.rename_remote_host(remote, username + "_dev_box", 'kave.io')
remote.run("mkdir -p /etc/kave/")
remote.run(
"/bin/echo http://repos:[email protected]/ >> /etc/kave/mirror"
)
lD.add_as_host(edit_remote=remote,
add_remote=remote,
dest_internal_ip=lA.priv_ip(iid))
lD.configure_keyless(remote,
remote,
dest_internal_ip=lA.priv_ip(iid),
preservehostname=True)
# This is not needed for Centos7
tos = remote.detect_linux_version()
lD.disable_security(remote)
lD.confallssh(remote, restart=False)
lD.confremotessh(remote, restart=False)
lD.confsshpermissions(remote)
vols = []
vols.append(
lA.add_new_ebs_vol(iid, {
"Mount": "/opt",
"Size": 10,
"Attach": "/dev/sdb"
}, keyloc))
vols.append(
lA.add_new_ebs_vol(iid, {
"Mount": "/var/log",
"Size": 2,
ret = ambari_get("clusters")
##################################################################
# Start ambari agents
##################################################################
print "Attempting to start ambari agents on all", len(hosts), "nodes"
sys.stdout.flush()
ambari = lD.remoteHost("root", thehost, access_key)
# Step one, install myself, dsh and deploy ambari agents to all nodes
lD.install_pdsh(ambari)
# modify iptables, only in case of Centos6
lD.disable_security(ambari)
admin = ambari.run("hostname")
whole_cluster = lD.multiremotes(hosts, jump=ambari)
# Check if all nodes in the cluster are contactable
try:
whole_cluster.check(firsttime=True)
except lD.ShellExecuteError:
print "Could not access machines with passwordless ssh, the ambari node must have passwordless ssh access to the " \
"rest, fix and try again"
raise
# Verify that all nodes have similar system times
ambtime = int(ambari.run("date -u '+%s'"))
cltime = whole_cluster.run("date -u '+%s'")
ret = ambari_get("clusters")
##################################################################
# Start ambari agents
##################################################################
print "Attempting to start ambari agents on all", len(hosts), "nodes"
sys.stdout.flush()
ambari = lD.remoteHost("root", thehost, access_key)
# Step one, install myself, dsh and deploy ambari agents to all nodes
lD.install_pdsh(ambari)
# modify iptables, only in case of Centos6
lD.disable_security(ambari)
admin = ambari.run("hostname")
whole_cluster = lD.multiremotes(hosts, jump=ambari)
# Check if all nodes in the cluster are contactable
try:
whole_cluster.check(firsttime=True)
except lD.ShellExecuteError:
print "Could not access machines with passwordless ssh, the ambari node must have passwordless ssh access to the " \
"rest, fix and try again"
raise
# Verify that all nodes have similar system times
ambtime = int(ambari.run("date -u '+%s'"))
cltime = whole_cluster.run("date -u '+%s'")
if instancegroup["AccessType"] == "admin":
# print "found group", instancegroup["Name"]
for instance in instancegroups[instancegroup["Name"]]:
# print "found instance"+instance
for otherinstance in instance_to_remote:
if otherinstance:
# give itself also keyless root access to itself!
lD.configure_keyless(instance_to_remote[instance],
instance_to_remote[otherinstance],
lA.priv_ip(otherinstance),
preservehostname=True)
print "=============================================="
print "Turn off SE linux and IPTables (yeah, I know)"
print "=============================================="
lD.disable_security(allremotes)
print "==================================="
print "add any extra disk space (parallelized per instance)"
print "==================================="
sys.stdout.flush()
adtoiids = []
admounts = []
for instancegroup in cluster_config["InstanceGroups"]:
if "ExtraDisks" in instancegroup:
for instance in instancegroups[instancegroup["Name"]]:
admounts.append(instancegroup["ExtraDisks"])
adtoiids.append(instance)
lA.add_ebs_volumes(adtoiids, admounts, amazon_keyfile)
# #for conf in instancegroup["ExtraDisks"]:
# # for instance in instancegroups[instancegroup["Name"]]:
resources = lA.find_all_child_resources(iid)
lA.tag_resources(resources, security_config["Tags"])
remote.register()
##################################################
# Renaming, configuring firewall and adding more disk space
##################################################
print "Renaming, configuring firewall and adding more disk space"
lD.rename_remote_host(remote, username + "_dev_box", 'kave.io')
remote.run("mkdir -p /etc/kave/")
remote.run("/bin/echo http://repos:[email protected]/ >> /etc/kave/mirror")
lD.add_as_host(edit_remote=remote, add_remote=remote, dest_internal_ip=lA.priv_ip(iid))
lD.configure_keyless(remote, remote, dest_internal_ip=lA.priv_ip(iid), preservehostname=True)
# This is not needed for Centos7
tos = remote.detect_linux_version()
lD.disable_security(remote)
lD.confallssh(remote, restart=False)
lD.confremotessh(remote, restart=False)
lD.confsshpermissions(remote)
vols = []
vols.append(lA.add_new_ebs_vol(iid, {"Mount": "/opt", "Size": 10, "Attach": "/dev/sdb"}, keyloc))
vols.append(lA.add_new_ebs_vol(iid, {"Mount": "/var/log", "Size": 2, "Attach": "/dev/sdc"}, keyloc))
vols.append(lA.add_new_ebs_vol(iid, {"Mount": "/home", "Size": 100, "Attach": "/dev/sdd"}, keyloc))
remote.describe()
if "Tags" in security_config:
lA.tag_resources(vols, security_config["Tags"])
print "OK, iid " + iid + " now lives at IP " + ip
##################################################
for instance in instancegroups[instancegroup["Name"]]:
# print "found instance"+instance
for otherinstance in instance_to_remote:
if otherinstance:
# give itself also keyless root access to itself!
lD.configure_keyless(instance_to_remote[instance], instance_to_remote[otherinstance],
lA.priv_ip(otherinstance), preservehostname=True)
print "=============================================="
print "Turn off SE linux and IPTables (yeah, I know)"
print "=============================================="
allremotes.run("yum install -y firewalld")
allremotes.run("systemctl enable firewalld")
allremotes.run("systemctl restart firewalld")
time.sleep(20)
lD.disable_security(allremotes)
print "==================================="
print "add any extra disk space (parallelized per instance)"
print "==================================="
sys.stdout.flush()
adtoiids = []
admounts = []
for instancegroup in cluster_config["InstanceGroups"]:
if "ExtraDisks" in instancegroup:
for instance in instancegroups[instancegroup["Name"]]:
admounts.append(instancegroup["ExtraDisks"])
adtoiids.append(instance)
lA.add_ebs_volumes(adtoiids, admounts, amazon_keyfile)
# #for conf in instancegroup["ExtraDisks"]:
# # for instance in instancegroups[instancegroup["Name"]]:
