Esempio n. 1
0
 def remote_from_cluster_stdout(self, stdout, mname='ambari'):
     """
     take the output from the up_aws_cluster script and parse it
     first check it was successful, then find one machine (mname) connect command
     so that I may return a remoteHost object and the iid
     remoteHost, iid
     """
     import kavedeploy as lD
     connectcmd = ""
     for line in range(len(stdout.split('\n'))):
         if (mname + " connect remotely with") in stdout.split("\n")[line]:
             connectcmd = stdout.split("\n")[line + 1].strip()
     adict = stdout.split("\n")[-2].replace("Complete, created:", "")
     # try interpreting as json
     adict = d2j(adict)
     iid, ip = adict[mname]
     self.assertTrue(ip in connectcmd, ip + " Wrong IP seen for connecting to " + mname + ' ' + connectcmd)
     jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
     import json
     acconf = json.loads(jsondat.read())
     jsondat.close()
     keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
     self.assertTrue(keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
                     "wrong keyfile seen in (" + connectcmd + ")")
     return lD.remoteHost("root", ip, keyfile), iid
Esempio n. 2
0
 def remote_from_cluster_stdout(self, stdout, mname='ambari'):
     """
     take the output from the up_aws_cluster script and parse it
     first check it was successful, then find one machine (mname) connect command
     so that I may return a remoteHost object and the iid
     remoteHost, iid
     """
     import kavedeploy as lD
     connectcmd = ""
     for line in range(len(stdout.split('\n'))):
         if (mname + " connect remotely with") in stdout.split("\n")[line]:
             connectcmd = stdout.split("\n")[line + 1].strip()
     adict = stdout.split("\n")[-2].replace("Complete, created:", "")
     # try interpreting as json
     adict = d2j(adict)
     iid, ip = adict[mname]
     self.assertTrue(
         ip in connectcmd, ip + " Wrong IP seen for connecting to " +
         mname + ' ' + connectcmd)
     jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
     import json
     acconf = json.loads(jsondat.read())
     jsondat.close()
     keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
     self.assertTrue(
         keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
         "wrong keyfile seen in (" + connectcmd + ")")
     return lD.remoteHost("root", ip, keyfile), iid
Esempio n. 3
0
    def deploy_os(self, osval, instancetype="m4.large"):
        """
        Up one centos machine with the scripts and return an lD.remoteHost to that machine
        instancetype -> None: m4.large
        """
        import kavedeploy as lD
        import kaveaws as lA
        instancetype = lA.chooseinstancetype(instancetype)
        deploy_dir = os.path.realpath(os.path.dirname(lD.__file__) + '/../')
        stdout = lD.run_quiet(deploy_dir + "/aws/deploy_known_instance.py "
                              + osval + " Test-" + osval + "-" + self.service + " "
                              + instancetype + " --not-strict")
        self.assertTrue(stdout.split("\n")[-1].startswith("OK, iid "))
        iid = stdout.split("\n")[-1].strip()[len("OK, iid "):].split(" ")[0]
        ip = stdout.split("\n")[-1].strip().split(" ")[-1]
        self.assertTrue(stdout.split("\n")[-3].startswith("connect remotely with:"))
        connectcmd = stdout.split("\n")[-2]
        self.assertTrue(ip in connectcmd, "wrong IP seen in (" + connectcmd + ")")
        jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
        import json

        acconf = json.loads(jsondat.read())
        jsondat.close()
        keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
        self.assertTrue(keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
                        "wrong keyfile seen in (" + connectcmd + ")")
        ambari = lD.remoteHost("root", ip, keyfile)
        ambari.register()
        import time
        time.sleep(5)
        if osval.startswith("Centos"):
            # add 10GB to /opt
            stdout = lD.run_quiet(
                deploy_dir + "/aws/add_ebsvol_to_instance.py " + iid + " --not-strict")
        return ambari, iid
Esempio n. 4
0
    def deploy_os(self, osval, instancetype="m4.large"):
        """
        Up one centos machine with the scripts and return an lD.remoteHost to that machine
        instancetype -> None: m4.large
        """
        import kavedeploy as lD
        import kaveaws as lA
        instancetype = lA.chooseinstancetype(instancetype)
        deploy_dir = os.path.realpath(os.path.dirname(lD.__file__) + '/../')
        stdout = lD.run_quiet(deploy_dir + "/aws/deploy_known_instance.py "
                              + osval + " Test-" + osval + "-" + self.service + " "
                              + instancetype + " --not-strict")
        self.assertTrue(stdout.split("\n")[-1].startswith("OK, iid "))
        iid = stdout.split("\n")[-1].strip()[len("OK, iid "):].split(" ")[0]
        ip = stdout.split("\n")[-1].strip().split(" ")[-1]
        self.assertTrue(stdout.split("\n")[-3].startswith("connect remotely with:"))
        connectcmd = stdout.split("\n")[-2]
        self.assertTrue(ip in connectcmd, "wrong IP seen in (" + connectcmd + ")")
        jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
        import json

        acconf = json.loads(jsondat.read())
        jsondat.close()
        keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
        self.assertTrue(keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
                        "wrong keyfile seen in (" + connectcmd + ")")
        ambari = lD.remoteHost("root", ip, keyfile)
        ambari.register()
        import time
        time.sleep(5)
        if osval.startswith("Centos"):
            # add 10GB to /opt
            stdout = lD.run_quiet(
                deploy_dir + "/aws/add_ebsvol_to_instance.py " + iid + " --not-strict")
        return ambari, iid
Esempio n. 5
0
    def deploy_dev(self, instancetype="m4.large"):
        """
        Up one centos machine with the scripts and return an lD.remoteHost to that machine
        instancetype -> None: m4.large
        """
        import kavedeploy as lD
        import kaveaws as lA
        instancetype = lA.chooseinstancetype(instancetype)

        deploy_dir = os.path.realpath(os.path.dirname(lD.__file__) + '/../')
        stdout = lD.run_quiet(deploy_dir +
                              "/aws/deploy_one_centos_instance.py Test-" +
                              self.service + " " + instancetype +
                              " --ambari-dev --not-strict")
        self.assertTrue(stdout.split("\n")[-1].startswith("OK, iid "))
        iid = stdout.split("\n")[-1].strip()[len("OK, iid "):].split(" ")[0]
        ip = stdout.split("\n")[-1].strip().split(" ")[-1]
        self.assertTrue(
            stdout.split("\n")[-3].startswith("connect remotely with:"))
        connectcmd = stdout.split("\n")[-2]
        self.assertTrue(ip in connectcmd,
                        "wrong IP seen in (" + connectcmd + ")")
        jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
        import json

        acconf = json.loads(jsondat.read())
        jsondat.close()
        keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
        self.assertTrue(
            keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
            "wrong keyfile seen in (" + connectcmd + ")")
        # add 10GB as /opt by default!
        ambari = lD.remoteHost("root", ip, keyfile)
        ambari.register()
        #
        # configure keyless access to itself! Needed for blueprints, but already done now by the new_dev_image script,
        #  but the internal ip will be different here!
        # lD.add_as_host(edit_remote=ambari,add_remote=ambari,dest_internal_ip=lA.priv_ip(iid)) #done in the deploy
        # script...
        #
        lD.configure_keyless(ambari,
                             ambari,
                             dest_internal_ip=lA.priv_ip(iid),
                             preservehostname=True)
        abranch = ""
        if self.branch:
            abranch = self.branch
        ambari.cp(
            os.path.realpath(os.path.dirname(lD.__file__)) +
            "/../remotescripts/default.netrc", "~/.netrc")
        return ambari, iid
Esempio n. 6
0
    def runTest(self):
        """
        Tests which cehck the function of the deployment library,
        but do not need any environment parameters or access to aws
        """
        import kavedeploy as lD

        lD.testproxy()
        self.assertIsNot(lD.which("ls"), None)
        self.assertRaises(RuntimeError, lD.run_quiet, ("thisisnotacommand"))
        stdout = lD.run_quiet(['which', 'ls'], shell=False)
        self.assertTrue('/bin/ls' in stdout)
        self.assertIsNot(lD.which("pdsh"), None,
                         "pdsh is not installed, please install it in order to test the multiremotes functionality, "
                         "sudo yum -y install pdsh")
        lD.run_quiet("touch /tmp/fake_test_ssh_key.pem")
        lD.run_quiet("chmod 400 /tmp/fake_test_ssh_key.pem")
        test = lD.remoteHost("root", "test", '/tmp/fake_test_ssh_key.pem')
        test = lD.multiremotes([test.host], access_key='/tmp/fake_test_ssh_key.pem')
Esempio n. 7
0
    def deploy_dev(self, instancetype="m4.large"):
        """
        Up one centos machine with the scripts and return an lD.remoteHost to that machine
        instancetype -> None: m4.large
        """
        import kavedeploy as lD
        import kaveaws as lA
        instancetype = lA.chooseinstancetype(instancetype)

        deploy_dir = os.path.realpath(os.path.dirname(lD.__file__) + '/../')
        stdout = lD.run_quiet(deploy_dir + "/aws/deploy_one_centos_instance.py Test-"
                              + self.service + " " + instancetype + " --ambari-dev --not-strict")
        self.assertTrue(stdout.split("\n")[-1].startswith("OK, iid "))
        iid = stdout.split("\n")[-1].strip()[len("OK, iid "):].split(" ")[0]
        ip = stdout.split("\n")[-1].strip().split(" ")[-1]
        self.assertTrue(stdout.split("\n")[-3].startswith("connect remotely with:"))
        connectcmd = stdout.split("\n")[-2]
        self.assertTrue(ip in connectcmd, "wrong IP seen in (" + connectcmd + ")")
        jsondat = open(os.path.expanduser(os.environ["AWSSECCONF"]))
        import json

        acconf = json.loads(jsondat.read())
        jsondat.close()
        keyfile = acconf["AccessKeys"]["SSH"]["KeyFile"]
        self.assertTrue(keyfile in connectcmd or os.path.expanduser(keyfile) in connectcmd,
                        "wrong keyfile seen in (" + connectcmd + ")")
        # add 10GB as /opt by default!
        ambari = lD.remoteHost("root", ip, keyfile)
        ambari.register()
        #
        # configure keyless access to itself! Needed for blueprints, but already done now by the new_dev_image script,
        #  but the internal ip will be different here!
        # lD.add_as_host(edit_remote=ambari,add_remote=ambari,dest_internal_ip=lA.priv_ip(iid)) #done in the deploy
        # script...
        #
        lD.configure_keyless(ambari, ambari, dest_internal_ip=lA.priv_ip(iid), preservehostname=True)
        abranch = ""
        if self.branch:
            abranch = self.branch
        ambari.cp(os.path.realpath(os.path.dirname(lD.__file__))
                  + "/../remotescripts/default.netrc",
                  "~/.netrc")
        return ambari, iid
Esempio n. 8
0
if __name__ == "__main__":
    ip, iid, security_conf, dest_type = parse_opts()
    # only needed in main function
    installfrom = os.path.realpath(os.sep.join(__file__.split(os.sep)[:-1]))
    liblocation = os.path.realpath(installfrom)
    jsondat = open(os.path.expanduser(security_conf))
    security_config = json.loads(jsondat.read())
    jsondat.close()
    sys.path.append(liblocation)
    import kavedeploy as lD
    import kaveaws as lA

    lA.checksecjson(security_config, requirefield=[], requirekeys=["SSH"])
    if ip is None:
        lA.testaws()
        ip = lA.pub_ip(iid)
    git = False
    gitenv = None
    if lD.detect_proxy():
        print "Did you already configure this machine to access port " + str(
            lD.proxy_port) + "? If not you'll need to turn your proxy off."
    lD.testproxy()
    remote = lD.remoteHost('root', ip, security_config["AccessKeys"]["SSH"]["KeyFile"])
    if "GIT" in security_config["AccessKeys"]:
        git = True
        gitenv = security_config["AccessKeys"]["GIT"]
    lD.deploy_our_soft(remote, pack="kavetoolbox", git=git,
                       gitenv=gitenv, options='--' + dest_type)
    # if dest_type == "workstation":
    #    lD.confremotessh(remote)
Esempio n. 9
0
print "submitted"
iid = lA.iid_from_up_json(upped)[0]
import time

time.sleep(5)
lA.name_resource(iid, username + "_dev_box")
ip = lA.pub_ip(iid)
acount = 0
while (ip is None and acount < 20):
    print "waiting for IP"
    lD.mysleep(1)
    ip = lA.pub_ip(iid)
    acount = acount + 1

remoteuser = lA.default_usernamedict[lA.default_os]
remote = lD.remoteHost(remoteuser, ip, keyloc)
print "waiting until contactable"
lD.wait_until_up(remote, 20)
remote = lD.remote_cp_authkeys(remote, 'root')
if "Tags" in security_config:
    resources = lA.find_all_child_resources(iid)
    lA.tag_resources(resources, security_config["Tags"])
remote.register()

##################################################
# Renaming, configuring firewall and adding more disk space
##################################################
print "Renaming, configuring firewall and adding more disk space"
lD.rename_remote_host(remote, username + "_dev_box", 'kave.io')
remote.run("mkdir -p /etc/kave/")
remote.run(
Esempio n. 10
0
def add_new_ebs_vol(iid, conf, access_key):
    """
    Create and name a new ebs volume, give it to a pre-existing instance and mount it on that instance

    conf is a dictionary which must contain:
    "Mount": "where-to_mount_it",   "Size" : SizeIngGB, "Attach" : "aws_expected_device_name", "Fdisk" :
    "device_name_seen_by_fdisk"
    e.g.:
    "Mount": "/opt2",   "Size" : 1, "Attach" : "/dev/sdb", "Fdisk" : "/dev/xvdb"
    Fdisk is optional, if not given it will be guessed from "Attach" and the region.
    i.e.:
    region  Attach  FDisk
    eu-*    sd<X>     xvd<X>   (e.g. sdb->xvdb)
    ap-*    sd<Y>     xvd<Y+4>  (e.g. sbd->xvdf)
    """
    try:
        i = desc_instance(iid)
    except lD.ShellExecuteError:
        raise ValueError(iid + " is not one of your instance IDs")
    # get a reference to this instance
    ip = pub_ip(iid)
    remote = lD.remoteHost("root", ip, access_key)
    # choose the fdisk device in the case this is a broken Tokyo centos6 instance
    if "Fdisk" not in conf:
        import string
        alpha = string.ascii_lowercase
        skip = 0
        if detect_region().startswith('eu'):
            # eu-*    sd<X>     xvd<X>   (e.g. sdb->xvdb)
            skip = 0
        elif detect_region().startswith(
                'ap') and remote.detect_linux_version() in ["Centos6"]:
            # ap-*    sd<Y>     xvd<Y+4>  (e.g. sbd->xvdf)
            skip = 4
        conf["Fdisk"] = '/dev/xvd' + alpha[alpha.index(conf["Attach"][-1]) +
                                           skip]

    av_zone = i["Reservations"][0]["Instances"][0]["Placement"][
        "AvailabilityZone"]
    voljson = runawstojson("ec2 create-volume --size " + str(conf["Size"]) +
                           " --availability-zone " + av_zone)
    instnam = ""
    for tag in i["Reservations"][0]["Instances"][0]["Tags"]:
        if tag["Key"] == "Name":
            instnam = tag["Value"]
    # print voljson
    vol_id = voljson["VolumeId"]
    name_resource(vol_id, instnam + conf["Mount"].replace("/", "_"))
    time.sleep(5)
    count = 0
    while count < 10:
        descvol = runawstojson("ec2 describe-volumes --volume " + vol_id)
        # print descvol
        if descvol['Volumes'][0]["State"] == "available":
            break
        time.sleep(5)
        count = count + 1
    resjson = runawstojson("ec2 attach-volume --volume-id " + vol_id +
                           " --instance-id " + iid + " --device " +
                           conf["Attach"])
    # print resjson
    time.sleep(5)
    count = 0
    while count < 10:
        descvol = runawstojson("ec2 describe-volumes --volume " + vol_id)
        # print descvol
        if descvol['Volumes'][0]['Attachments'][0]["State"] == "attached":
            break
        time.sleep(5)
        count = count + 1
    remote.cp(
        os.path.dirname(__file__) + "/../remotescripts/fdiskwrap.sh",
        "~/fdiskwrap.sh")
    remote.run("chmod a+x fdiskwrap.sh")
    try:
        remote.run("./fdiskwrap.sh " + conf["Fdisk"])
    except lD.ShellExecuteError:
        time.sleep(30)
        remote.run("./fdiskwrap.sh " + conf["Fdisk"])
    remote.run("mkfs.ext4 -b 4096 " + conf["Fdisk"] + "1 ")
    remote.run("bash -c 'echo \"" + conf["Fdisk"] + "1 " + conf["Mount"] +
               " ext4 defaults 1 1\" >> /etc/fstab'")
    mvto = " /" + conf["Mount"].replace("/", "_")
    remote.run("bash -c \"mv " + conf["Mount"] + mvto + "; mkdir " +
               conf["Mount"] + "; mount " + conf["Mount"] + ";\"")
    remote.run("bash -c \"if [ -d " + mvto + " ] ; then chmod --reference " +
               mvto + " " + conf["Mount"] + " ; fi\"")
    remote.run("bash -c 'shopt -s dotglob; if [ \"$(ls -A " + mvto +
               ")\" ] ; then mv " + mvto + "/* " + conf["Mount"] + "/ ; fi'")
    res = remote.run("df -hP")
    if conf["Mount"] not in res:
        raise lD.ShellExecuteError(
            "Could not mount the requested disk, resulted in " + res)
    return vol_id
Esempio n. 11
0
    url = prot + ambhost + ':' + str(port) + api + apath
    # print url
    s = lD.request_session()
    req = s.post(url, auth=HTTPBasicAuth(user, passwd), headers={
        'X-Requested-By': 'ambari'}, data=json.dumps(data))
    return _r2j(req)

ret = ambari_get("clusters")
##################################################################
# Start ambari agents
##################################################################

print "Attempting to start ambari agents on all", len(hosts), "nodes"
sys.stdout.flush()

ambari = lD.remoteHost("root", thehost, access_key)

# Step one, install myself, dsh and deploy ambari agents to all nodes
lD.install_pdsh(ambari)

# modify iptables, only in case of Centos6
lD.disable_security(ambari)
admin = ambari.run("hostname")

whole_cluster = lD.multiremotes(hosts, jump=ambari)

# Check if all nodes in the cluster are contactable
try:
    whole_cluster.check(firsttime=True)
except lD.ShellExecuteError:
    print "Could not access machines with passwordless ssh, the ambari node must have passwordless ssh access to the " \
Esempio n. 12
0
    liblocation = os.path.realpath(installfrom)
    jsondat = open(os.path.expanduser(security_conf))
    security_config = json.loads(jsondat.read())
    jsondat.close()
    sys.path.append(liblocation)
    import kavedeploy as lD
    import kaveaws as lA

    lA.checksecjson(security_config, requirefield=[], requirekeys=["SSH"])
    if ip is None:
        lA.testaws()
        ip = lA.pub_ip(iid)
    git = False
    gitenv = None
    if lD.detect_proxy():
        print "Did you already configure this machine to access port " + str(
            lD.proxy_port) + "? If not you'll need to turn your proxy off."
    lD.testproxy()
    remote = lD.remoteHost('root', ip,
                           security_config["AccessKeys"]["SSH"]["KeyFile"])
    if "GIT" in security_config["AccessKeys"]:
        git = True
        gitenv = security_config["AccessKeys"]["GIT"]
    lD.deploy_our_soft(remote,
                       pack="kavetoolbox",
                       git=git,
                       gitenv=gitenv,
                       options='--' + dest_type)
    # if dest_type == "workstation":
    #    lD.confremotessh(remote)
Esempio n. 13
0
        ip = None
        acount = 0
        while (ip is None and acount < 10):
            try:
                ip = lA.pub_ip(iid)
            except ValueError:
                pass
            if ip is None:
                print "waiting for IP"
                lD.mysleep(1)
            acount = acount + 1
        if ip is None:
            raise SystemError(iid + " no ip assigned after quite some time")

        remoteuser = lA.default_usernamedict[lA.default_os]
        remote = lD.remoteHost(remoteuser, ip, amazon_keyfile)

        lD.wait_until_up(remote, 20)
        remote = lD.remote_cp_authkeys(remote, 'root')
        if "Tags" in security_config:
            resources = lA.find_all_child_resources(iid)
            lA.tag_resources(resources, security_config["Tags"])
        remote.register()
        instance_to_remote[iid] = remote

allremotes = [
    "ssh:root@" + remote.host for remote in instance_to_remote.values()
]
allremotes = lD.multiremotes(list_of_hosts=allremotes,
                             access_key=amazon_keyfile)
print "test local PDSH, install pdcp"
Esempio n. 14
0
def add_new_ebs_vol(iid, conf, access_key):
    """
    Create and name a new ebs volume, give it to a pre-existing instance and mount it on that instance

    conf is a dictionary which must contain:
    "Mount": "where-to_mount_it",   "Size" : SizeIngGB, "Attach" : "aws_expected_device_name", "Fdisk" :
    "device_name_seen_by_fdisk"
    e.g.:
    "Mount": "/opt2",   "Size" : 1, "Attach" : "/dev/sdb", "Fdisk" : "/dev/xvdb"
    Fdisk is optional, if not given it will be guessed from "Attach" and the region.
    i.e.:
    region  Attach  FDisk
    eu-*    sd<X>     xvd<X>   (e.g. sdb->xvdb)
    ap-*    sd<Y>     xvd<Y+4>  (e.g. sbd->xvdf)
    """
    try:
        i = desc_instance(iid)
    except lD.ShellExecuteError:
        raise ValueError(iid + " is not one of your instance IDs")
    # get a reference to this instance
    ip = pub_ip(iid)
    remote = lD.remoteHost("root", ip, access_key)
    # choose the fdisk device in the case this is a broken Tokyo centos6 instance
    if "Fdisk" not in conf:
        import string
        alpha = string.ascii_lowercase
        skip = 0
        if detect_region().startswith('eu'):
            # eu-*    sd<X>     xvd<X>   (e.g. sdb->xvdb)
            skip = 0
        elif detect_region().startswith('ap') and remote.detect_linux_version() in ["Centos6"]:
            # ap-*    sd<Y>     xvd<Y+4>  (e.g. sbd->xvdf)
            skip = 4
        conf["Fdisk"] = '/dev/xvd' + alpha[alpha.index(conf["Attach"][-1]) + skip]

    av_zone = i["Reservations"][0]["Instances"][0]["Placement"]["AvailabilityZone"]
    voljson = runawstojson("ec2 create-volume --size " + str(conf["Size"]) + " --availability-zone " + av_zone)
    instnam = ""
    for tag in i["Reservations"][0]["Instances"][0]["Tags"]:
        if tag["Key"] == "Name":
            instnam = tag["Value"]
    # print voljson
    vol_id = voljson["VolumeId"]
    name_resource(vol_id, instnam + conf["Mount"].replace("/", "_"))
    time.sleep(5)
    count = 0
    while count < 10:
        descvol = runawstojson("ec2 describe-volumes --volume " + vol_id)
        # print descvol
        if descvol['Volumes'][0]["State"] == "available":
            break
        time.sleep(5)
        count = count + 1
    resjson = runawstojson(
        "ec2 attach-volume --volume-id " + vol_id + " --instance-id " + iid + " --device " + conf["Attach"])
    # print resjson
    time.sleep(5)
    count = 0
    while count < 10:
        descvol = runawstojson("ec2 describe-volumes --volume " + vol_id)
        # print descvol
        if descvol['Volumes'][0]['Attachments'][0]["State"] == "attached":
            break
        time.sleep(5)
        count = count + 1
    remote.cp(os.path.dirname(__file__) + "/../remotescripts/fdiskwrap.sh", "~/fdiskwrap.sh")
    remote.run("chmod a+x fdiskwrap.sh")
    try:
        remote.run("./fdiskwrap.sh " + conf["Fdisk"])
    except lD.ShellExecuteError:
        time.sleep(30)
        remote.run("./fdiskwrap.sh " + conf["Fdisk"])
    remote.run("mkfs.ext4 -b 4096 " + conf["Fdisk"] + "1 ")
    remote.run("bash -c 'echo \"" + conf["Fdisk"] + "1 " + conf["Mount"] + " ext4 defaults 1 1\" >> /etc/fstab'")
    mvto = " /" + conf["Mount"].replace("/", "_")
    remote.run("bash -c \"mv " + conf["Mount"] + mvto + "; mkdir " + conf["Mount"] + "; mount " + conf["Mount"] + ";\"")
    remote.run("bash -c \"if [ -d " + mvto + " ] ; then chmod --reference " + mvto + " " + conf["Mount"] + " ; fi\"")
    remote.run("bash -c 'shopt -s dotglob; if [ \"$(ls -A " + mvto + ")\" ] ; then mv " + mvto + "/* " + conf[
        "Mount"] + "/ ; fi'")
    res = remote.run("df -hP")
    if conf["Mount"] not in res:
        raise lD.ShellExecuteError("Could not mount the requested disk, resulted in " + res)
    return vol_id
Esempio n. 15
0
        ip = None
        acount = 0
        while (ip is None and acount < 10):
            try:
                ip = lA.pub_ip(iid)
            except ValueError:
                pass
            if ip is None:
                print "waiting for IP"
                lD.mysleep(1)
            acount = acount + 1
        if ip is None:
            raise SystemError(iid + " no ip assigned after quite some time")

        remoteuser = lA.default_usernamedict[lA.default_os]
        remote = lD.remoteHost(remoteuser, ip, amazon_keyfile)

        lD.wait_until_up(remote, 20)
        remote = lD.remote_cp_authkeys(remote, 'root')
        if "Tags" in security_config:
            resources = lA.find_all_child_resources(iid)
            lA.tag_resources(resources, security_config["Tags"])
        remote.register()
        instance_to_remote[iid] = remote

allremotes = ["ssh:root@" + remote.host for remote in instance_to_remote.values()]
allremotes = lD.multiremotes(list_of_hosts=allremotes, access_key=amazon_keyfile)
print "test local PDSH, install pdcp"
print allremotes.run("echo yes")
allremotes.run("yum clean all")
lD.install_pdsh(allremotes)
Esempio n. 16
0
    print "submitted"
    iid = lA.iid_from_up_json(upped)[0]
    import time

    time.sleep(5)
    lA.name_resource(iid, "new-dev-image")
    ip = lA.pub_ip(iid)
    acount = 0
    while (ip is None and acount < 20):
        print "waiting for IP"
        lD.mysleep(1)
        ip = lA.pub_ip(iid)
        acount = acount + 1

    remoteuser = lA.default_usernamedict[lA.default_os]
    remote = lD.remoteHost(remoteuser, ip, keyloc)
    print "waiting until contactable"
    lD.wait_until_up(remote, 20)
    remote = lD.remote_cp_authkeys(remote, 'root')
    if "Tags" in security_config:
        resources = lA.find_all_child_resources(iid)
        lA.tag_resources(resources, security_config["Tags"])
    remote.register()
    print "Renaming, configuring firewall and adding more disk space"
    lD.rename_remote_host(remote, "ambari", 'kave.io')
    remote.run("mkdir -p /etc/kave/")
    remote.run("/bin/echo http://repos:[email protected]/ >> /etc/kave/mirror")
    remote.run('yum install curl nss -y; yum update curl nss -y --enablerepo="updates"')
    lD.add_as_host(edit_remote=remote, add_remote=remote, dest_internal_ip=lA.priv_ip(iid))
    lD.configure_keyless(remote, remote, dest_internal_ip=lA.priv_ip(iid), preservehostname=True)
    # nope! Don't want 443 as ssh by default any longer!
acount = 0
while (ip is None and acount < 20):
    print "waiting for IP"
    lD.mysleep(1)
    ip = lA.pub_ip(iid)
    acount = acount + 1

if osval == "Centos6":
    remoteuser = '******'
else:
    remoteuser = ''.join([i for i in osval if not i.isdigit()]).lower()

if os.path.exists(os.path.realpath(os.path.expanduser(keyloc))):
    print "waiting until contactable, ctrl-C to quit"
    try:
        remote = lD.remoteHost(remoteuser, ip, keyloc)
        lD.wait_until_up(remote, 20)
        if "Tags" in security_config:
            resources = lA.find_all_child_resources(iid)
            lA.tag_resources(resources, security_config["Tags"])
        remote.register()
        remote = lD.remote_cp_authkeys(remote, 'root')
        lD.rename_remote_host(remote, machinename, 'kave.io')
        lD.confallssh(remote)
        if osval.startswith("Centos"):
            remote.run("yum clean all")
        remote.describe()
    except KeyboardInterrupt:
        pass
else:
    print "Warning: not contactable since keyfile supplied does not exist locally,",
Esempio n. 18
0
    req = s.post(url,
                 auth=HTTPBasicAuth(user, passwd),
                 headers={'X-Requested-By': 'ambari'},
                 data=json.dumps(data))
    return _r2j(req)


ret = ambari_get("clusters")
##################################################################
# Start ambari agents
##################################################################

print "Attempting to start ambari agents on all", len(hosts), "nodes"
sys.stdout.flush()

ambari = lD.remoteHost("root", thehost, access_key)

# Step one, install myself, dsh and deploy ambari agents to all nodes
lD.install_pdsh(ambari)

# modify iptables, only in case of Centos6
lD.disable_security(ambari)
admin = ambari.run("hostname")

whole_cluster = lD.multiremotes(hosts, jump=ambari)

# Check if all nodes in the cluster are contactable
try:
    whole_cluster.check(firsttime=True)
except lD.ShellExecuteError:
    print "Could not access machines with passwordless ssh, the ambari node must have passwordless ssh access to the " \