Exemple #1
0
def create(clusterConfigFilePath, overrideClusterId):
    '''
    return:
    {
       "dns": "172.17.0.2",
       "hosts": "172.17.0.2 master\n172.17.0.3 slave1\n172.17.04 slave2"
    }
    '''

    if not isInitialized():
        print "initialize first"
        return 1

    dnsServerAddress = None
    hosts = ""

    with open(clusterConfigFilePath, "r") as conffile:
        conf = conffile.read()

    try:
        clusterConfig = json.loads(conf)
    except ValueError as e:
        print "Given cluster config json file " + clusterConfigFilePath + " is invalid "
        print e.message
        return 1

    # docker build if Dockerfile is specified
    clusterConfig = _flattenDockerfile(clusterConfig)

    clusterConfig = _flattenHostname(clusterConfig)

    if overrideClusterId != None:
        clusterConfig["id"] = overrideClusterId

    # Append DNS
    dnsNode = {
        "hostname": "dclouddns",
        "imageName": REPO_DNS_BASE,
        "cmd": "service sshd start && tail -f /var/log/yum.log"
    }
    clusterConfig["nodes"].insert(0, dnsNode)

    for i in range(len(clusterConfig["nodes"])):
        # The first iteration is for DNS
        node = clusterConfig["nodes"][i]

        container_name = _generateContainerName(clusterConfig["id"],
                                                node["hostname"])

        cmd = [
            "docker",
            "run",
            "-d"  # daemon
            ,
            "--privileged"
        ]

        # DNS
        cmd.append("--dns")
        if i == 0:
            cmd.append("127.0.0.1")  # localhost
        else:
            cmd.append(dnsServerAddress)

        if "dns" in clusterConfig:
            for dnsIp in clusterConfig["dns"]:
                cmd.append("--dns")
                cmd.append(dnsIp)

        cmd.append("--name")
        cmd.append(container_name)

        fqdn = node["hostname"] + "." + clusterConfig["domain"]
        cmd.append("-h")
        cmd.append(fqdn)

        if "volumes" in node:
            for volumn in node["volumes"]:
                cmd.append("-v")
                cmd.append(volumn)

        cmd.append(node["imageName"])
        cmd.append("bash")
        cmd.append("-c")
        cmd.append(node["cmd"])
        print "executing: " + ' '.join(cmd)
        subprocess.call(cmd)

        ip = docker.getContainerIpAddress(container_name)
        if i == 0:
            dnsServerAddress = ip
        hosts += ip + " " + fqdn + " " + node["hostname"] + "\n"

    print "dnsServerAddress: " + dnsServerAddress
    if (not ssh.connection_check(dnsServerAddress, "root", "changeme")):
        print "**** ERROR ****"
        print "ssh connection to root@" + dnsServerAddress + " could not be established"
        return 1

    ssh.exec_command2(dnsServerAddress, "root", "changeme",
                      "echo '" + hosts + "' > /etc/dcloud/dnsmasq/hosts")
    ssh.exec_command2(dnsServerAddress, "root", "changeme",
                      "service dnsmasq restart")

    print "hosts:"
    print hosts
    result = RunResult()
    result.dns = dnsServerAddress
    result.hosts = hosts
    return 0
Exemple #2
0
def create(clusterConfigFilePath, overrideClusterId):
    '''
    return:
    {
       "dns": "172.17.0.2",
       "hosts": "172.17.0.2 master\n172.17.0.3 slave1\n172.17.04 slave2"
    }
    '''
    
    if not isInitialized():
        print "initialize first"
        return 1
    
    dnsServerAddress = None
    hosts = ""
    
    with open(clusterConfigFilePath, "r") as conffile:
        conf = conffile.read()
    
    try:
        clusterConfig = json.loads(conf)
    except ValueError as e:
        print "Given cluster config json file " + clusterConfigFilePath + " is invalid "
        print e.message
        return 1
        
    # docker build if Dockerfile is specified
    clusterConfig = _flattenDockerfile(clusterConfig)

    clusterConfig = _flattenHostname(clusterConfig)
    
    if overrideClusterId != None:
        clusterConfig["id"] = overrideClusterId

    # Append DNS
    dnsNode = {
        "hostname" : "dclouddns",
        "imageName" : REPO_DNS_BASE,
        "cmd" : "service sshd start && tail -f /var/log/yum.log"
    }
    clusterConfig["nodes"].insert(0, dnsNode)

    for i in range(len(clusterConfig["nodes"])):
        # The first iteration is for DNS
        node = clusterConfig["nodes"][i]

        container_name = _generateContainerName(clusterConfig["id"], node["hostname"])

        cmd = ["docker", "run"
		    , "-d" # daemon
      		, "--privileged"]

        # DNS
        cmd.append("--dns")
        if i == 0:
            cmd.append("127.0.0.1") # localhost 
        else:
            cmd.append(dnsServerAddress)

        if "dns" in clusterConfig:
            for dnsIp in clusterConfig["dns"]:
                cmd.append("--dns")
                cmd.append(dnsIp)

        cmd.append("--name")
        cmd.append(container_name)

        fqdn = node["hostname"] + "." + clusterConfig["domain"]
        cmd.append("-h")
        cmd.append(fqdn)

        if "volumes" in node:
            for volumn in node["volumes"]:
                cmd.append("-v")
                cmd.append(volumn)

        cmd.append(node["imageName"])
        cmd.append("bash")
        cmd.append("-c")
        cmd.append(node["cmd"])
        print "executing: " + ' '.join(cmd)
        subprocess.call(cmd)

        ip = docker.getContainerIpAddress(container_name)
        if i == 0:
            dnsServerAddress = ip
        hosts += ip + " " + fqdn + " " + node["hostname"] + "\n"

    print "dnsServerAddress: " + dnsServerAddress
    if(not ssh.connection_check(dnsServerAddress, "root", "changeme")):
        print "**** ERROR ****"
        print "ssh connection to root@" + dnsServerAddress + " could not be established"
        return 1

    ssh.exec_command2(dnsServerAddress, "root", "changeme", "echo '" + hosts + "' > /etc/dcloud/dnsmasq/hosts")
    ssh.exec_command2(dnsServerAddress, "root", "changeme", "service dnsmasq restart")

    print "hosts:"
    print hosts
    result = RunResult()
    result.dns = dnsServerAddress
    result.hosts = hosts
    return 0
Exemple #3
0
def ambariBase(clusterConfigFilePath, overrideClusterId,nodeCnt):
   
    print "Ambari Cluster Build Start Time: "+str(time.asctime(time.localtime(time.time())))
   
    dnsServerAddress = None
    hosts = ""
    hadoopHosts=[]
    rootPassword = "******"
    encPassword = crypt.crypt(rootPassword,"salt")


    with open(clusterConfigFilePath, "r") as conffile:
        conf = conffile.read()

    try:
        clusterConfig = json.loads(conf)
        clusterConfig["nodes"][0]["hostname"] = str(clusterConfig["nodes"][0]["hostname"]).replace("6",nodeCnt)

    except ValueError as e:
        print "Given cluster config json file " + clusterConfigFilePath + " is invalid "
        print e.message
        return 1

    # docker build if Dockerfile is specified
    clusterConfig = _flattenDockerfile(clusterConfig)
    clusterConfig = _flattenHostname(clusterConfig)

    if overrideClusterId != None:
        clusterConfig["id"] = overrideClusterId

    # Append DNS
    dnsNode = {
        "hostname" : "dclouddns",
        "imageName" : REPO_DNS_BASE,
        "cmd" : "service sshd start && tail -f /var/log/yum.log"
    }


    clusterConfig["nodes"].insert(0, dnsNode)
    clusterConfig = _flattenHostname(clusterConfig)

    ambariPath = "/root/software"
    volumes = ['/mnt/ambari']

    volumeBinds = {
        ambariPath : {'bind' : '/mnt/ambari', "ro" : False}
        }
    
 
    print "Setting Images for Use"
    pullImages(dnsNode["imageName"])

    for i in range(len(clusterConfig["nodes"])):
        dnsList=[]

        node = clusterConfig["nodes"][i]
        containerName = _generateContainerName(clusterConfig["id"], node["hostname"])
        #containerName = str(clusterConfig["id"])+"."+str(node["hostname"])
        domainName = clusterConfig["domain"]
        fqdn = node["hostname"] + "." + clusterConfig["domain"]

        cmdString = "bash -c '"+node["cmd"]+"'"
        dockerClient=dockerpy.Client()
        #containerId = dockerClient.create_container(node["imageName"],command=cmdString,hostname=node["hostname"],domainname=domainName,detach=True,name=containerName,volumes=volumes)["Id"]
        #containerId = dockerClient.create_container(node["imageName"],command=cmdString,hostname=node["hostname"]+"."+domainName,domainname=domainName,detach=True,name=containerName,volumes=volumes)["Id"]
        containerId = dockerClient.create_container(node["imageName"],command=cmdString,hostname=node["hostname"]+"."+domainName,detach=True,name=containerName,volumes=volumes)["Id"]
        print containerId
        if i == 0:
            dnsList.append("127.0.0.1")
        else:
            dnsList.append(dnsServerAddress)

        if "dns" in clusterConfig:
            for dnsIp in clusterConfig["dns"]:
                dnsList.append(dnsIp)

        dockerClient.start(containerId,dns=dnsList,dns_search=domainName,privileged=True,binds=volumeBinds)

        containerInfo = dockerClient.inspect_container(containerId)
        containerIP = containerInfo['NetworkSettings']['IPAddress']
        if i == 0:
            dnsServerAddress = containerIP

        hosts += containerIP + " " + fqdn + " " + node["hostname"] + "\n"

        if (node["hostname"] != "dclouddns"):
            hadoopHosts.append({"hostname":node["hostname"],"ip":containerIP,"fqdn":fqdn,"id":containerId})

    print "DNS Server Address: " + dnsServerAddress
    if(not ssh.connection_check(dnsServerAddress, "root", "changeme")):
        print "**** ERROR ****"
        print "ssh connection to root@" + dnsServerAddress + " could not be established"
        return 1

    ssh.exec_command2(dnsServerAddress, "root", "changeme", "echo '" + hosts + "' > /etc/dcloud/dnsmasq/hosts")
    ssh.exec_command2(dnsServerAddress, "root", "changeme", "service dnsmasq restart")

    print "Cluster Hosts:"
    print "-----------------------------------------"
    lines = hosts.split('\n')
    print "Management Host: " + lines[1]
    print "Hadoop Nodes:"
    lineCnt = 0
    for line in lines:
        if lineCnt > 1 :
            print line
        lineCnt+=1


    result = RunResult()
    result.dns = dnsServerAddress
    result.hosts = hosts


     
    #print "gpadmin user created on "+host["hostname"]
    #print "gpadmin users created"
    #print "Sharing Root SSH Keys Across Cluster"
    #shareSSHKeys(clusterConfig["id"],"root",rootPassword)
    #print "Sharing Root SSH Keys Completed"


    mgmtContainerHostname = dockerDriver.getContainerId(clusterConfig["nodes"][1]["hostname"])
    mgmtIPaddress = hadoopHosts[0]["ip"]

    print "Sharing root keys across Cluster"
    shareSSHKeys(clusterConfig["id"],"root",rootPassword)
    print  "Sharing root keys Completed"

    
  
    print "Cluster Build End Time: "+str(time.asctime(time.localtime(time.time())))
    print "Update Docker Host /etc/hosts for hostname based access"
    hostsfile(clusterConfig["id"],"/etc/hosts")
    if (os.path.isfile("/root/.ssh/known_hosts")):
        os.remove("/root/.ssh/known_hosts")
    phdDesigner.preAmbariSetup("root","changeme",hadoopHosts,clusterConfig["id"] ,nodeCnt)

    return 0