def analyzeHawqTables(master, database, username, password, emailAddress=""):
loggerInfo = buildReportLogger("analyze")
reportName = loggerInfo[0]
report = loggerInfo[1]
header = []
startString = "Analyze Database Tables to Generate Statistics"
uniInfoLog(startString, report)
header = "Analyzing HAWQ Tables"
uniInfoLog(header, report)
analyzeCMD = "analyzedb -d " + database + " -a -p 10"
ssh.exec_command2(master, username, password, analyzeCMD)
# hawqURI = queries.uri(master, port=5432, dbname=database, user=username, password=password)
# with queries.Session(hawqURI) as session:
# uniInfoLog("Analyze Dimension Tables", report)
# for table in dimensionTables:
# ddlString = "Analyze " + table
# startTime = datetime.datetime.now()
# uniInfoLog("Start " + ddlString + ": " + str(startTime), report)
# result = session.query(ddlString)
# stopTime = datetime.datetime.now()
# resultString = "Completed " + ddlString + ": " + str(stopTime) + " Elapsed Time: " + str(
# stopTime - startTime)
# uniInfoLog(resultString, report)
# if emailAddress:
# Email.sendEmail(emailAddress, ddlString + " Complete", resultString)
# uniInfoLog("Analyze Fact Tables", report)
#
# for table in factTables:
# ddlString = "analyze " + table
# startTime = datetime.datetime.now()
# uniInfoLog("Start " + ddlString + ": " + str(startTime), report)
# result = session.query(ddlString)
# stopTime = datetime.datetime.now()
# resultString = "Completed " + ddlString + ": " + str(stopTime) + " Elapsed Time: " + str(
# stopTime - startTime)
# uniInfoLog(resultString, report)
# if emailAddress:
# Email.sendEmail(emailAddress, ddlString + " Complete", resultString)
#
# if (emailAddress):
# messageLines = []
# with open(reportName, "r") as reportMsg:
# for line in reportMsg.readlines():
# messageLines.append(line)
# message = " ".join(messageLines)
# Email.sendEmail(emailAddress, "Table Analyze Final Report: " + (reportName.split('/')[2])[:-4], message)
if (emailAddress):
Email.sendEmail(emailAddress, "Table Analyze Final Report: " + (reportName.split('/')[2])[:-4],
"AnalyzeDB Complete")
def getAdminCreds(hostFile, adminUser):
password = getpass.getpass("Password for " + adminUser + " (needed for Buffer Clears):")
with open(hostFile, "r") as hostsReader:
hosts = hostsReader.readlines()
for host in hosts:
print "Testing access to " + host
try:
ssh.exec_command2(host.rstrip(), adminUser, password, "touch /.test")
except paramiko.AuthenticationException as e:
print "Error: Username/password not correct"
exit()
except Exception as e2:
print "Error: Unknown Host"
exit()
return password
def preAmbariSetup(username,password,hadoopHosts,clusterId,nodeCnt):
print "Configuring system for Ambari Install"
print "Installing Ambari Agent"
for host in hadoopHosts:
if "hadoop0" not in host["hostname"]:
print "Installing Ambari Agent on " + str(host['fqdn']) + " and Configuring for Server"
ssh.exec_command2(host['fqdn'],username,password,"rpm -ivh /mnt/ambari/*agent*")
#ssh.exec_command2(host['fqdn'],username,password,"rpm -ivh /mnt/ambari/*log4j*")
ssh.exec_command2(host['fqdn'],username,password,"sed -i s/^hostname=.*/hostname="+hadoopHosts[0]['fqdn']+"/ /etc/ambari-agent/conf/ambari-agent.ini")
ssh.exec_command2(host['fqdn'],username,password,"service ambari-agent start")
else:
ssh.exec_command2(host['fqdn'],username,password,"yum -y install postgresql-server")
ssh.exec_command2(host['fqdn'],username,password,"rpm -ivh /mnt/ambari/*server*")
def initializeHAWQ(username,password,hadoopHosts,clusterId,nodeCnt):
print "Initializing HAWQ..."
#print username
#
# print password
masterNode = clusterLayouts["clusterSize"][int(nodeCnt)][int(nodeCnt)]["hqm"][0]
standbyNode = clusterLayouts["clusterSize"][int(nodeCnt)][int(nodeCnt)]["hqm"][1]
hawqMaster = hadoopHosts[masterNode]["fqdn"]
hawqStandbyMaster = hadoopHosts[standbyNode]["fqdn"]
for host in hadoopHosts:
ssh.exec_command2(host["ip"],"root","changeme","sysctl -p")
print "HAWQ Master: "+hawqMaster
print "HAWQ StandbyMaster: "+hawqStandbyMaster
ssh.exec_command2(hadoopHosts[masterNode]["ip"],username,password,"source /usr/local/hawq/greenplum_path.sh")
ssh.exec_command2(hadoopHosts[masterNode]["ip"],username,password,"/etc/init.d/hawq init")
def create(clusterConfigFilePath, overrideClusterId):
'''
return:
{
"dns": "172.17.0.2",
"hosts": "172.17.0.2 master\n172.17.0.3 slave1\n172.17.04 slave2"
}
'''
if not isInitialized():
print "initialize first"
return 1
dnsServerAddress = None
hosts = ""
with open(clusterConfigFilePath, "r") as conffile:
conf = conffile.read()
try:
clusterConfig = json.loads(conf)
except ValueError as e:
print "Given cluster config json file " + clusterConfigFilePath + " is invalid "
print e.message
return 1
# docker build if Dockerfile is specified
clusterConfig = _flattenDockerfile(clusterConfig)
clusterConfig = _flattenHostname(clusterConfig)
if overrideClusterId != None:
clusterConfig["id"] = overrideClusterId
# Append DNS
dnsNode = {
"hostname" : "dclouddns",
"imageName" : REPO_DNS_BASE,
"cmd" : "service sshd start && tail -f /var/log/yum.log"
}
clusterConfig["nodes"].insert(0, dnsNode)
for i in range(len(clusterConfig["nodes"])):
# The first iteration is for DNS
node = clusterConfig["nodes"][i]
container_name = _generateContainerName(clusterConfig["id"], node["hostname"])
cmd = ["docker", "run"
, "-d" # daemon
, "--privileged"]
# DNS
cmd.append("--dns")
if i == 0:
cmd.append("127.0.0.1") # localhost
else:
cmd.append(dnsServerAddress)
if "dns" in clusterConfig:
for dnsIp in clusterConfig["dns"]:
cmd.append("--dns")
cmd.append(dnsIp)
cmd.append("--name")
cmd.append(container_name)
fqdn = node["hostname"] + "." + clusterConfig["domain"]
cmd.append("-h")
cmd.append(fqdn)
if "volumes" in node:
for volumn in node["volumes"]:
cmd.append("-v")
cmd.append(volumn)
cmd.append(node["imageName"])
cmd.append("bash")
cmd.append("-c")
cmd.append(node["cmd"])
print "executing: " + ' '.join(cmd)
subprocess.call(cmd)
ip = docker.getContainerIpAddress(container_name)
if i == 0:
dnsServerAddress = ip
hosts += ip + " " + fqdn + " " + node["hostname"] + "\n"
print "dnsServerAddress: " + dnsServerAddress
if(not ssh.connection_check(dnsServerAddress, "root", "changeme")):
print "**** ERROR ****"
print "ssh connection to root@" + dnsServerAddress + " could not be established"
return 1
ssh.exec_command2(dnsServerAddress, "root", "changeme", "echo '" + hosts + "' > /etc/dcloud/dnsmasq/hosts")
ssh.exec_command2(dnsServerAddress, "root", "changeme", "service dnsmasq restart")
print "hosts:"
print hosts
result = RunResult()
result.dns = dnsServerAddress
result.hosts = hosts
return 0
def create(clusterConfigFilePath, overrideClusterId):
'''
return:
{
"dns": "172.17.0.2",
"hosts": "172.17.0.2 master\n172.17.0.3 slave1\n172.17.04 slave2"
}
'''
if not isInitialized():
print "initialize first"
return 1
dnsServerAddress = None
hosts = ""
with open(clusterConfigFilePath, "r") as conffile:
conf = conffile.read()
try:
clusterConfig = json.loads(conf)
except ValueError as e:
print "Given cluster config json file " + clusterConfigFilePath + " is invalid "
print e.message
return 1
# docker build if Dockerfile is specified
clusterConfig = _flattenDockerfile(clusterConfig)
clusterConfig = _flattenHostname(clusterConfig)
if overrideClusterId != None:
clusterConfig["id"] = overrideClusterId
# Append DNS
dnsNode = {
"hostname": "dclouddns",
"imageName": REPO_DNS_BASE,
"cmd": "service sshd start && tail -f /var/log/yum.log"
}
clusterConfig["nodes"].insert(0, dnsNode)
for i in range(len(clusterConfig["nodes"])):
# The first iteration is for DNS
node = clusterConfig["nodes"][i]
container_name = _generateContainerName(clusterConfig["id"],
node["hostname"])
cmd = [
"docker",
"run",
"-d" # daemon
,
"--privileged"
]
# DNS
cmd.append("--dns")
if i == 0:
cmd.append("127.0.0.1") # localhost
else:
cmd.append(dnsServerAddress)
if "dns" in clusterConfig:
for dnsIp in clusterConfig["dns"]:
cmd.append("--dns")
cmd.append(dnsIp)
cmd.append("--name")
cmd.append(container_name)
fqdn = node["hostname"] + "." + clusterConfig["domain"]
cmd.append("-h")
cmd.append(fqdn)
if "volumes" in node:
for volumn in node["volumes"]:
cmd.append("-v")
cmd.append(volumn)
cmd.append(node["imageName"])
cmd.append("bash")
cmd.append("-c")
cmd.append(node["cmd"])
print "executing: " + ' '.join(cmd)
subprocess.call(cmd)
ip = docker.getContainerIpAddress(container_name)
if i == 0:
dnsServerAddress = ip
hosts += ip + " " + fqdn + " " + node["hostname"] + "\n"
print "dnsServerAddress: " + dnsServerAddress
if (not ssh.connection_check(dnsServerAddress, "root", "changeme")):
print "**** ERROR ****"
print "ssh connection to root@" + dnsServerAddress + " could not be established"
return 1
ssh.exec_command2(dnsServerAddress, "root", "changeme",
"echo '" + hosts + "' > /etc/dcloud/dnsmasq/hosts")
ssh.exec_command2(dnsServerAddress, "root", "changeme",
"service dnsmasq restart")
print "hosts:"
print hosts
result = RunResult()
result.dns = dnsServerAddress
result.hosts = hosts
return 0
def ambariBase(clusterConfigFilePath, overrideClusterId,nodeCnt):
print "Ambari Cluster Build Start Time: "+str(time.asctime(time.localtime(time.time())))
dnsServerAddress = None
hosts = ""
hadoopHosts=[]
rootPassword = "******"
encPassword = crypt.crypt(rootPassword,"salt")
with open(clusterConfigFilePath, "r") as conffile:
conf = conffile.read()
try:
clusterConfig = json.loads(conf)
clusterConfig["nodes"][0]["hostname"] = str(clusterConfig["nodes"][0]["hostname"]).replace("6",nodeCnt)
except ValueError as e:
print "Given cluster config json file " + clusterConfigFilePath + " is invalid "
print e.message
return 1
# docker build if Dockerfile is specified
clusterConfig = _flattenDockerfile(clusterConfig)
clusterConfig = _flattenHostname(clusterConfig)
if overrideClusterId != None:
clusterConfig["id"] = overrideClusterId
# Append DNS
dnsNode = {
"hostname" : "dclouddns",
"imageName" : REPO_DNS_BASE,
"cmd" : "service sshd start && tail -f /var/log/yum.log"
}
clusterConfig["nodes"].insert(0, dnsNode)
clusterConfig = _flattenHostname(clusterConfig)
ambariPath = "/root/software"
volumes = ['/mnt/ambari']
volumeBinds = {
ambariPath : {'bind' : '/mnt/ambari', "ro" : False}
}
print "Setting Images for Use"
pullImages(dnsNode["imageName"])
for i in range(len(clusterConfig["nodes"])):
dnsList=[]
node = clusterConfig["nodes"][i]
containerName = _generateContainerName(clusterConfig["id"], node["hostname"])
#containerName = str(clusterConfig["id"])+"."+str(node["hostname"])
domainName = clusterConfig["domain"]
fqdn = node["hostname"] + "." + clusterConfig["domain"]
cmdString = "bash -c '"+node["cmd"]+"'"
dockerClient=dockerpy.Client()
#containerId = dockerClient.create_container(node["imageName"],command=cmdString,hostname=node["hostname"],domainname=domainName,detach=True,name=containerName,volumes=volumes)["Id"]
#containerId = dockerClient.create_container(node["imageName"],command=cmdString,hostname=node["hostname"]+"."+domainName,domainname=domainName,detach=True,name=containerName,volumes=volumes)["Id"]
containerId = dockerClient.create_container(node["imageName"],command=cmdString,hostname=node["hostname"]+"."+domainName,detach=True,name=containerName,volumes=volumes)["Id"]
print containerId
if i == 0:
dnsList.append("127.0.0.1")
else:
dnsList.append(dnsServerAddress)
if "dns" in clusterConfig:
for dnsIp in clusterConfig["dns"]:
dnsList.append(dnsIp)
dockerClient.start(containerId,dns=dnsList,dns_search=domainName,privileged=True,binds=volumeBinds)
containerInfo = dockerClient.inspect_container(containerId)
containerIP = containerInfo['NetworkSettings']['IPAddress']
if i == 0:
dnsServerAddress = containerIP
hosts += containerIP + " " + fqdn + " " + node["hostname"] + "\n"
if (node["hostname"] != "dclouddns"):
hadoopHosts.append({"hostname":node["hostname"],"ip":containerIP,"fqdn":fqdn,"id":containerId})
print "DNS Server Address: " + dnsServerAddress
if(not ssh.connection_check(dnsServerAddress, "root", "changeme")):
print "**** ERROR ****"
print "ssh connection to root@" + dnsServerAddress + " could not be established"
return 1
ssh.exec_command2(dnsServerAddress, "root", "changeme", "echo '" + hosts + "' > /etc/dcloud/dnsmasq/hosts")
ssh.exec_command2(dnsServerAddress, "root", "changeme", "service dnsmasq restart")
print "Cluster Hosts:"
print "-----------------------------------------"
lines = hosts.split('\n')
print "Management Host: " + lines[1]
print "Hadoop Nodes:"
lineCnt = 0
for line in lines:
if lineCnt > 1 :
print line
lineCnt+=1
result = RunResult()
result.dns = dnsServerAddress
result.hosts = hosts
#print "gpadmin user created on "+host["hostname"]
#print "gpadmin users created"
#print "Sharing Root SSH Keys Across Cluster"
#shareSSHKeys(clusterConfig["id"],"root",rootPassword)
#print "Sharing Root SSH Keys Completed"
mgmtContainerHostname = dockerDriver.getContainerId(clusterConfig["nodes"][1]["hostname"])
mgmtIPaddress = hadoopHosts[0]["ip"]
print "Sharing root keys across Cluster"
shareSSHKeys(clusterConfig["id"],"root",rootPassword)
print "Sharing root keys Completed"
print "Cluster Build End Time: "+str(time.asctime(time.localtime(time.time())))
print "Update Docker Host /etc/hosts for hostname based access"
hostsfile(clusterConfig["id"],"/etc/hosts")
if (os.path.isfile("/root/.ssh/known_hosts")):
os.remove("/root/.ssh/known_hosts")
phdDesigner.preAmbariSetup("root","changeme",hadoopHosts,clusterConfig["id"] ,nodeCnt)
return 0
def icmInstall(mgmtServer,cfgFile,username,password,hadoopHosts,clusterId):
print "ICM Install on "+ str(mgmtServer)
#print cfgFile
pccPath = configFile.getParam(cfgFile,"PHD","pcc")
#filename = ssh.putFile(mgmtServer,pccPath,username,password)
fileName = os.path.split(pccPath)[1]
print "PCC Filename: "+fileName
ssh.exec_command2(mgmtServer,username,password,"yum -y install tar")
print "Untarring: "+"tar xvfz /mnt/pcc/"+fileName
ssh.exec_command2(mgmtServer,username,password,"cd /tmp;tar xvfz /mnt/pcc/"+fileName)
print "Installing PCC..."
#ssh.exec_command2(mgmtServer,username,password,"echo \"echo not running.\" > /etc/init.d/iptables")
#print ssh.exec_command2(mgmtServer,username,password,"/tmp/"+filename[:-7]+"/install")
ssh.exec_command2(mgmtServer,username,password,"/tmp/PCC*/install")
print "Install Complete"
print "Transfer Configuration: "
# #tar File, send across, untar
# basePath = os.path.split(__file__)[0]
# tar = tarfile.open("/tmp/clusterConfig.tar","w")
# fileList = []
# for root, subFolders, files in os.walk(str(basePath)+"/template/"):
# for file in files:
# fullPath = os.path.join(root,file)
# relativePath = ((root.replace(str(basePath),""))+"/"+file)[1:]
# tar.add(fullPath,relativePath)
# tar.close()
#
#
#
# ssh.putFile(mgmtServer,"/tmp/clusterConfig.tar",username,password)
# ssh.exec_command2(mgmtServer,username,password,"cd /tmp;tar xvf /tmp/clusterConfig.tar")
# ICM Setup
phdPath = configFile.getParam(cfgFile,"PHD","phd")
#fileName = ssh.putFile(mgmtServer,phdPath,username,password)
fileName = os.path.split(phdPath)[1]
print "PHD Filename: "+fileName
hawqPath = configFile.getParam(cfgFile,"PHD","pads")
hawqFileName = os.path.split(hawqPath)[1]
print "HAWQ Filename: "+ hawqFileName
print "tar xvfz /tmp/"+fileName
ssh.exec_command2(mgmtServer,username,password,"cd /tmp;tar xvfz /mnt/pcc/"+fileName)
print "untar complete"
print "tar xvfz /tmp/"+hawqFileName
ssh.exec_command2(mgmtServer,username,password,"cd /tmp;tar xvfz /mnt/pcc/"+hawqFileName)
print "untar complete"
#ssh.exec_command2(mgmtServer,username,password,"echo \"changeme\" | passwd --stdin gpadmin")
print "changed GPADMIN password"
print "Start Import of /mnt/pcc/"+fileName[:-7]
# import stack details
stackConfig = {
"stack_name": "PHD-2.1.0.0",
"stack_properties": {
"rpm_rel": "46",
"hive_package_version": "0.12.0_gphd_3_1_0_0-175",
"hadoop_rpm_version": "2.2.0_gphd_3_1_0_0-175",
"rpm_label": "gphd",
"hbase_package_version": "0.96.0_gphd_3_1_0_0-175",
"zookeeper_package_version": "3.4.5_gphd_3_1_0_0-175",
"rpm_version": "2_0_2_0",
"pig_package_version": "0.12.0_gphd_3_1_0_0-175",
"mahout_package_version": "0.7_gphd_3_1_0_0-175"
},
"stack_type": "phd"
}
# apiClient.importStack(stackConfig)
ssh.exec_command2(mgmtServer,"gpadmin","changeme","icm_client import -s /tmp/"+fileName[:-7])
print "Import Complete"
print "Start Import of /mnt/pcc/"+hawqFileName[:-7]
ssh.exec_command2(mgmtServer,"gpadmin","changeme","icm_client import -s /tmp/"+hawqFileName[:-7])
print "Starting Cluster Deploy..."
# RUN PREPAREHOSTS VIA API
inputData={}
hostList = []
hostCnt = 0
for host in hadoopHosts:
if hostCnt > 0:
hostList.append(host["hostname"])
hostCnt+=1
inputData["hosts"] = hostList
inputData["jdkPath"] = ""
inputData["verbose"] = True
inputData["setupNTP"] = False
inputData["ntpServer"] = ""
inputData["disableSELinux"] = False
inputData["disableIPTables"] = False
inputData["sysConfigDir"] = ""
inputData["skipPasswordlessSSH"] = False
inputData["rootPassword"]= password
inputData["gpadminPassword"]= password
inputData["gphdStackVer"]="PHD-2.1.0.0"
inputData["clusterId"] = clusterId
# Get OAuth file
ssh.getFile(mgmtServer,"/etc/gphd/gphdmgr/conf/oauth2-clients.conf","/tmp/oauth2-clients.conf",username,password)
apiClient = APIClient (mgmtServer)
apiClient.prepareHosts(inputData)
# Thread the deploy and then look through install status.
#clusterConfigJSON = json.dumps(open(str(basePath)+"/template/clusterConfig.xml","r").readlines())
#print APIClient.deployConfiguration(apiClient,clusterConfigJSON,inputData)
#print apiClient.getClusterTemplateJson(inputData)
ssh.exec_command2(mgmtServer,"gpadmin","changeme","icm_client deploy -s -p -c /mnt/config")
print "Cluster Deploy Complete: PCC is available at https://"+mgmtServer+":5443"
print "Starting Cluster..."
ssh.exec_command2(mgmtServer,"gpadmin","changeme","icm_client start -l phdcluster")
print "Cluster Started"
#print apiClient.getClusterStatus(inputData)