def resetNode(cls, nodename, hostlist,action):
HADOOP_HOME = Config.get('hadoop', 'HADOOP_HOME')
HADOOP_BIN = os.path.join(HADOOP_HOME,"bin")
nodetype = { 'namenode': 'master',
'datanode': 'slave',
'jobtracker': 'master',
'tasktracker': 'slave',
}[nodename]
MAPRED_USER = Config.get('hadoop', 'MAPRED_USER')
HDFS_USER = Config.get('hadoop', 'HDFS_USER')
user = { 'namenode': HDFS_USER,
'datanode': MAPRED_USER,
'jobtracker': HDFS_USER,
'tasktracker': MAPRED_USER,
}[nodename]
cmd = os.path.join(HADOOP_BIN,action + "-" + nodetype + ".cmd")
pattern = "(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
for host in hostlist:
if not re.search(pattern,host):
host = host.split(':')[0]
host = socket.gethostbyname(host.replace('\n','').strip())
host = re.findall(pattern, host)[0]
logger.info( action + " " + nodename + " on host " + host)
Machine.runinbackgroundAs(user, cmd, host,HADOOP_BIN)
def getTTHostForAttemptId(cls,attemptID):
HADOOP_JOBTRACKER_LOG = Config.get('hadoop', 'HADOOP_JOBTRACKER_LOG')
f = open(HADOOP_JOBTRACKER_LOG,"r")
for line in f:
searchFor = re.search(".*" + attemptID + ".*tracker_.*/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*",line)
if searchFor != None:
return searchFor.group(1)
def getDatanodeCount(cls):
exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -report")
if exit_code == 0:
m = re.match(".*Datanodes available: (\d+) \(", output, re.DOTALL)
if m:
return int(m.group(1))
return 0
def triggerSleepJob(cls,numOfMaps,numOfReduce,mapsleeptime,reducesleeptime,numOfJobs,queue='',background=False):
jobCounter = 0
while (jobCounter < numOfJobs):
sleepCmd = " jar " + Config.get('hadoop', 'HADOOP_EXAMPLES_JAR') + " sleep " + queue + " -m " + numOfMaps + " -r " + numOfReduce + " -mt " + mapsleeptime +" -rt " + reducesleeptime
if background:
Hadoop.runInBackground(sleepCmd)
else:
Hadoop.run(sleepCmd)
jobCounter = jobCounter + 1
def generateTestReportConf(infile, outfile, results):
config = ConfigParser()
config.optionxform=str
config.read(infile)
if config.has_section(SECTION):
for option, value in config.items(SECTION):
if value != "": continue
elif option == "BUILD_ID" and config.has_option(SECTION, "REPO_URL"):
config.set(SECTION, option, getBuildId(config.get(SECTION, "REPO_URL")))
config.remove_option(SECTION, "REPO_URL")
elif option == "HOSTNAME":
config.set(SECTION, option, socket.getfqdn())
elif option == "COMPONENT_VERSION":
if not config.has_option(SECTION, "COMPONENT") or config.get(SECTION, "COMPONENT") == "":
config.set(SECTION, "COMPONENT", "Hadoop")
config.set(SECTION, option, getComponentVersion(config.get(SECTION, "COMPONENT")))
elif option == "OS":
config.set(SECTION, option, platform.platform())
elif option == "SECURE" and Config.hasOption('hadoop', 'IS_SECURE'):
config.set(SECTION, option, Config.get('hadoop', 'IS_SECURE').lower())
elif option == "BLOB":
pass
elif option == "RAN":
config.set(SECTION, option, results[0] + len(results[1]))
elif option == "PASS":
config.set(SECTION, option, results[0])
elif option == "FAIL":
config.set(SECTION, option, len(results[1]))
elif option == "SKIPPED":
config.set(SECTION, option, results[2])
elif option == "ABORTED":
config.set(SECTION, option, results[3])
elif option == "FAILED_TESTS":
config.set(SECTION, option, ",".join(results[1]))
elif option == "SINGLE_NODE":
from beaver.component.hadoop import HDFS
if HDFS.getDatanodeCount() > 1:
config.set(SECTION, option, "false")
else:
config.set(SECTION, option, "true")
config.write(open(outfile, 'w'))
def getDruidUser(cls):
'''
Returns Druid user
'''
return Config.get('druid', 'DRUID_USER')
def getHadoopExamplesJar(cls):
return Config.get('hadoop', 'HADOOP_EXAMPLES_JAR')
def getConfigValue(cls, propertyValue, defaultValue=None):
return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "core-site.xml"), propertyValue, defaultValue=defaultValue)
def getLocalDirInfo(host):
return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "mapred-site.xml"), "mapred.local.dir")
def exitSafemode(cls):
exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -safemode leave")
if re.search(".*Safe mode is OFF",output) != None:
return True
else:
return False
def runInBackgroundAs(cls, user, cmd, cwd=None, env=None):
flume_cmd = Config.get('flume-ng', 'FLUME_CMD')
flume_cmd += " " + cmd
return Machine.runinbackgroundAs(user, flume_cmd, cwd=cwd, env=env)
def switch_master_version(cls, action, version, config=None):
'''
Switches Hive master services' version
:param action: Whether to "upgrade" or "downgrade"
:param version: Version to be switched to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from beaver.component.hive import Hive
currentHiveVersion = Hive.getVersion()
if action == 'upgrade':
# Backup the database used by the Hive Metastore
logger.info(
"Performing backup of the Hive Metastore DB before starting the upgrade"
)
Hive.backupMetastoreDB(cls._metastore_backup_file)
node = Hive.getHiveHost()
# Stop the old Hive Metastore
logger.info("Stopping the Hive Metastore")
Hive.stopService(services=["metastore"])
# Upgrade Hive Metastore servers to new version
hdpSelect.changeVersion("hive-metastore", version, node)
if action == 'upgrade':
logger.info("Upgrading the Hive metastore schema")
Hive.upgradeSchema()
# Restart Hive Metastore servers one at a time
logger.info("Restarting the Hive Metastore")
Hive.startService(services=["metastore"])
# Start new Hive Server 2 instance
confHS2Port = Hive.getHiveserver2ThriftPort()
hs2port = util.getNextAvailablePort(node, confHS2Port)
hdpSelect.changeVersion("hive-server2", version, node)
Hive.modifyConfig(config,
services=['hiveserver2'],
restartService=False)
logger.info(
"Starting a new HiveServer2 at port '%d' for assisting rolling-upgrade"
% hs2port)
if hs2port != confHS2Port:
changes = {'hive-site.xml': {'hive.server2.thrift.port': hs2port}}
Hive.modifyConfig(changes,
services=["hiveserver2"],
restartService=False)
Hive.startService(services=["hiveserver2"])
cls._hs2_live_ports = [Hive.getHiveserver2ThriftPort(), hs2port]
# Deregister the old Hive Server 2 instances
logger.info("Deregistering the HiveServer2 on version '%s'" %
currentHiveVersion)
Hive.deregisterHiveServer2(version=currentHiveVersion)
from beaver.component.hcatalog import Hcatalog
# Stop the old WebHCat server
logger.info("Stopping the WebHCat server")
node = Config.get('templeton',
'TEMPLETON_HOST',
default=Machine.getfqdn())
webhcatPort = Config.get('templeton',
'TEMPLETON_PORT',
default="50111")
# Stop the old WebHCat server
logger.info("Stop the WebHCat server")
Hcatalog.stop(node)
# Upgrade WebHCat to the new version
hdpSelect.changeVersion("hive-webhcat", version, node)
# Start the WebHCat server
logger.info("Restarting the WebHCat server")
newConfDir = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
'localWebhcatConf')
if os.path.exists(newConfDir):
Hcatalog.start(node, hcat_confdir=newConfDir)
else:
Hcatalog.start(node)
import os
import random
import ssl
import tempfile
import threading
import time
import urllib
import urllib2
from beaver.component.hadoop import Hadoop
from beaver.config import Config
logger = logging.getLogger(__name__)
KNOX_GUEST_USER = "******"
KNOX_GUEST_PWD = "guest-password"
KNOX_HOST = Config.get('knox', 'KNOX_HOST').split(',')[0] # Just use the first Knox instance in the list for now.
KNOX_PORT = "8443"
HDFS_USER = Config.get('hadoop', 'HDFS_USER')
WEBHDFS_CONTEXT_ROOT = "/gateway/sandbox/webhdfs/v1"
if Hadoop.isAmbari():
WEBHDFS_CONTEXT_ROOT = "/gateway/default/webhdfs/v1"
class KnoxWebHdfsLoadGenerator(object):
"""
A class that generates WebHDFS load though the Knox gateway.
Note: This currently does not support AD authentication.
The following operations are currently performed in a loop by the load generator in each of the load threads.
1. Mkdir
2. Rename dir
def run_background_job(cls, runSmokeTestSetup=False, config=None):
'''
Runs background long running Hive Job
:param runSmokeTestSetup: Runs smoke test setup if set to true
:param config: expected configuration location
:return: Total number of long running jobs started
'''
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][BGJob] Long running job for Hive component started")
setqueue = ""
if Hive.isTezEnabled():
setqueue = "set tez.queue.name=%s; " % cls._yarn_queue
else:
setqueue = "set mapred.job.queue.name=%s; " % cls._yarn_queue
logger.info("**** Running Hive CLI Test ****")
query = setqueue + " create table if not exists hive_cli_lr (a string); select sleep(%d, 2000, 'hdfs://%s/hive_cli_lr', 'hdfs://%s/END') from (select count(*) from hive_cli_lr) a;" % (
cls._max_bgjtest_duration, cls._hdfs_bgjtest_dir,
cls._hdfs_bgjtest_dir)
Hive.runQuery(query, background=True)
logger.info("**** Running Beeline CLI Test ****")
# Create the sleep function within the same Beeline session
# Function created outside of HS2 instance are not picked
query = setqueue + "\n"
query += "drop function sleep2;\n"
query += "create function sleep2 as 'org.apache.hive.udf.generic.GenericUDFSleep' using jar 'hdfs://%s/hive-udfs-0.1.jar';\n" % cls._hdfs_bgjtest_dir
query += "create table if not exists bline_cli_lr (a string);\n"
query += "select sleep2(%d, 2000, 'hdfs://%s/bline_cli_lr', 'hdfs://%s/END') from (select count(*) from bline_cli_lr) a;\n" % (
cls._max_bgjtest_duration, cls._hdfs_bgjtest_dir,
cls._hdfs_bgjtest_dir)
Hive.runQueryOnBeeline(query, readFromFile=True, background=True)
logger.info("**** Running WebHCat Test ****")
webhcatHost = Config.get('templeton',
'TEMPLETON_HOST',
default=Machine.getfqdn())
webhcatPort = Config.get('templeton',
'TEMPLETON_PORT',
default="50111")
url = "http://%s:%s/templeton/v1/hive" % (webhcatHost, webhcatPort)
query = setqueue + " set mapred.task.timeout=0; create table if not exists whcat_rest_lr (a string); select sleep(%d, 2000, 'hdfs://%s/whcat_rest_lr', 'hdfs://%s/END') from (select count(*) from whcat_rest_lr) a;" % (
cls._max_bgjtest_duration, cls._hdfs_bgjtest_dir,
cls._hdfs_bgjtest_dir)
params = {'execute': query}
status_code, stdout = util.curl(url, method='POST', params=params)
retry = 0
while status_code == 404 and retry < 3:
time.sleep(15)
status_code, stdout = util.curl(url, method='POST', params=params)
retry += 1
if status_code != 200:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJobSetup] Long running job for WebHCat failed due to status code = %d"
% status_code)
logger.error(
"Webhcat request failed with the following error: %s\n" %
stdout)
if runSmokeTestSetup:
logger.info("**** Running Hive Smoke Test Setup ****")
cls.smoke_test_setup()
return 3
class ruHive:
_metastore_backup_file = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
"metastore_backup.sql")
_yarn_queue = "hive"
_hdfs_smoketest_dir = "/tmp/hivetest"
_smoketest_tbl = "smtesttbl"
_hdfs_bgjtest_dir = "/tmp/hivelrtest"
_bgjtest_tbl = "lrtesttbl"
_hdfs_user = Config.get("hadoop", 'HDFS_USER')
_num_of_webhcat_bgj = 1
_num_of_rows_lr = 10000
_num_of_rows_smoke = 100
_max_bgjtest_duration = 72000000
_hs2_live_ports = []
_shortbgj_hive_process = None
_shortbgj_bline_process = None
@classmethod
def background_job_setup(cls, runSmokeTestSetup=False, config=None):
'''
Setup for background long running job
:param runSmokeTestSetup: Runs smoke test setup if set to true
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][BGJob] Long running test setup for Hive started")
cls.doBackgroundJobSetup(cls._hdfs_bgjtest_dir)
UpgradePerNode.reportProgress(
"[INFO][Hive][BGJob] Long running test setup for Hive finished")
if runSmokeTestSetup:
logger.info("**** Running Hive Smoke Test Setup ****")
cls.smoke_test_setup()
@classmethod
def smoke_test_setup(cls):
'''
Setup required to run Smoke test
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][Smoke] Smoke test setup for Hive started")
cls.doSetup(cls._hdfs_smoketest_dir, cls._smoketest_tbl,
cls._num_of_rows_smoke, "smoke")
UpgradePerNode.reportProgress(
"[INFO][Hive][Smoke] Smoke test setup for Hive finished")
@classmethod
def doBackgroundJobSetup(cls, hdfs_test_dir):
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
logger.info("Preparing the test setup for Hive background job")
udfjar = os.path.join(Config.getEnv('WORKSPACE'), "tests", "hive",
"hive-udf", "hive-udfs-0.1.jar")
HDFS.createDirectory(hdfs_test_dir,
user=cls._hdfs_user,
perm='777',
force=True)
HDFS.copyFromLocal(udfjar, hdfs_test_dir)
query = "drop function sleep; create function sleep as 'org.apache.hive.udf.generic.GenericUDFSleep' using jar 'hdfs://%s/hive-udfs-0.1.jar';" % hdfs_test_dir
exit_code, stdout = Hive.runQuery(query)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Setup] Long running failed due to exitcode = %d"
% exit_code)
else:
UpgradePerNode.reportProgress(
"[PASSED][Hive][Setup] Long running finished successfully")
@classmethod
def doSetup(cls, hdfs_test_dir, tbl_name, num_of_rows, type):
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
logger.info("Generating test table dataset with %d rows" % num_of_rows)
test_data_file = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
tbl_name + ".dat")
f = open(test_data_file, 'w')
userid = 100000
for i in xrange(num_of_rows):
for j in range(random.randint(3, 8)):
f.write("%d|%d\n" % (userid + i, random.randint(10, 80)))
f.close()
hdfs_tbl_dir = hdfs_test_dir + "/" + tbl_name
logger.info("Copying the test dataset to HDFS directory '%s'" %
hdfs_tbl_dir)
HDFS.createDirectory(hdfs_test_dir,
user=cls._hdfs_user,
perm='777',
force=True)
HDFS.createDirectory(hdfs_tbl_dir, perm='777')
HDFS.copyFromLocal(test_data_file, hdfs_tbl_dir)
HDFS.chmod(cls._hdfs_user, '777', hdfs_tbl_dir)
logger.info("Creating table '%s' and verification tables" % tbl_name)
query = "drop table if exists %s;\n" % tbl_name
query += "create external table %s (userid string, age int) row format delimited fields terminated by '|' stored as textfile location '%s';\n" % (
tbl_name, hdfs_tbl_dir)
query += "drop table if exists %s_hive_verify;\n" % tbl_name
query += "create table %s_hive_verify (userid string, age int);\n" % tbl_name
if type == "Long running":
for i in range(cls._num_of_webhcat_bgj):
query += "drop table if exists %s_wh_%d;\n" % (tbl_name, i + 1)
query += "create table %s_wh_%d (userid string, age int);\n" % (
tbl_name, i + 1)
hivesetupfile = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
"hivesetup.sql")
util.writeToFile(query, hivesetupfile)
exit_code, stdout = Hive.run("-f " + hivesetupfile, logoutput=False)
if type:
msg = "%s job setup for Hive component" % type
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Setup] %s failed due to exitcode = %d" %
(msg, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][Hive][Setup] %s finished successfully" % msg)
@classmethod
def run_background_job(cls, runSmokeTestSetup=False, config=None):
'''
Runs background long running Hive Job
:param runSmokeTestSetup: Runs smoke test setup if set to true
:param config: expected configuration location
:return: Total number of long running jobs started
'''
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][BGJob] Long running job for Hive component started")
setqueue = ""
if Hive.isTezEnabled():
setqueue = "set tez.queue.name=%s; " % cls._yarn_queue
else:
setqueue = "set mapred.job.queue.name=%s; " % cls._yarn_queue
logger.info("**** Running Hive CLI Test ****")
query = setqueue + " create table if not exists hive_cli_lr (a string); select sleep(%d, 2000, 'hdfs://%s/hive_cli_lr', 'hdfs://%s/END') from (select count(*) from hive_cli_lr) a;" % (
cls._max_bgjtest_duration, cls._hdfs_bgjtest_dir,
cls._hdfs_bgjtest_dir)
Hive.runQuery(query, background=True)
logger.info("**** Running Beeline CLI Test ****")
# Create the sleep function within the same Beeline session
# Function created outside of HS2 instance are not picked
query = setqueue + "\n"
query += "drop function sleep2;\n"
query += "create function sleep2 as 'org.apache.hive.udf.generic.GenericUDFSleep' using jar 'hdfs://%s/hive-udfs-0.1.jar';\n" % cls._hdfs_bgjtest_dir
query += "create table if not exists bline_cli_lr (a string);\n"
query += "select sleep2(%d, 2000, 'hdfs://%s/bline_cli_lr', 'hdfs://%s/END') from (select count(*) from bline_cli_lr) a;\n" % (
cls._max_bgjtest_duration, cls._hdfs_bgjtest_dir,
cls._hdfs_bgjtest_dir)
Hive.runQueryOnBeeline(query, readFromFile=True, background=True)
logger.info("**** Running WebHCat Test ****")
webhcatHost = Config.get('templeton',
'TEMPLETON_HOST',
default=Machine.getfqdn())
webhcatPort = Config.get('templeton',
'TEMPLETON_PORT',
default="50111")
url = "http://%s:%s/templeton/v1/hive" % (webhcatHost, webhcatPort)
query = setqueue + " set mapred.task.timeout=0; create table if not exists whcat_rest_lr (a string); select sleep(%d, 2000, 'hdfs://%s/whcat_rest_lr', 'hdfs://%s/END') from (select count(*) from whcat_rest_lr) a;" % (
cls._max_bgjtest_duration, cls._hdfs_bgjtest_dir,
cls._hdfs_bgjtest_dir)
params = {'execute': query}
status_code, stdout = util.curl(url, method='POST', params=params)
retry = 0
while status_code == 404 and retry < 3:
time.sleep(15)
status_code, stdout = util.curl(url, method='POST', params=params)
retry += 1
if status_code != 200:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJobSetup] Long running job for WebHCat failed due to status code = %d"
% status_code)
logger.error(
"Webhcat request failed with the following error: %s\n" %
stdout)
if runSmokeTestSetup:
logger.info("**** Running Hive Smoke Test Setup ****")
cls.smoke_test_setup()
return 3
@classmethod
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run smoke test for hive
:param smoketestnumber: Used for unique output log location
'''
return cls.run_client_smoketest(config=config)
@classmethod
def background_job_teardown(cls):
'''
Cleanup for long running Hive jobs
'''
from beaver.component.hive import Hive
logger.info(
"Make sure to switch the HiveServer2 to use the default port")
adminUser = Machine.getAdminUser()
hiveHost = Hive.getHiveHost()
for port in cls._hs2_live_ports:
pid = Machine.getPIDByPort(port, host=hiveHost, user=adminUser)
if pid:
Machine.killProcessRemote(pid, host=hiveHost, user=adminUser)
time.sleep(2)
if len(cls._hs2_live_ports) > 0:
Hive.startService(services=["hiveserver2"])
@classmethod
def stopHiveLongRunningJob(cls):
'''
Stop the long running background queries
'''
logger.info("Push the file to HDFS to signal END of queries")
HDFS.touchz(cls._hdfs_bgjtest_dir + "/END")
@classmethod
def verifyLongRunningJob(cls):
'''
Validate long running background job after end of all component upgrade
'''
# Assumption is all the MR jobs for Hive are alive and running
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
logger.info(
"Verify the query triggered by Hive CLI is alive post upgrade")
passed = cls.verifyLongRunningQuery("hive_cli_lr")
if passed:
UpgradePerNode.reportProgress(
"[PASSED][Hive][BGJob] Long running test for Hive Metastore passed"
)
else:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Long running test for Hive Metastore failed"
)
logger.info(
"Verify the query triggered by Beeline CLI is alive post upgrade")
passed = cls.verifyLongRunningQuery("bline_cli_lr")
if passed:
UpgradePerNode.reportProgress(
"[PASSED][Hive][BGJob] Long running test for HiveServer2 passed"
)
else:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Long running test for HiveServer2 failed"
)
logger.info(
"Verify the query triggered by WebHCat is alive post upgrade")
passed = cls.verifyLongRunningQuery("whcat_rest_lr")
if passed:
UpgradePerNode.reportProgress(
"[PASSED][Hive][BGJob] Long running test for WebHCat passed")
else:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Long running test for WebHCat failed")
@classmethod
def verifyLongRunningQuery(cls, file_to_verify):
lfile = os.path.join(Config.getEnv('ARTIFACTS_DIR'), file_to_verify)
exit_code, stdout = HDFS.copyToLocal(
cls._hdfs_bgjtest_dir + "/" + file_to_verify, lfile)
if exit_code != 0:
logger.info("Error fetching the timestamp file from HDFS")
return False
lines = open(lfile, 'r').readlines()
if len(lines) == 0:
logger.info("Empty timestamp file")
return False
try:
ts = int(lines[-1])
# Shutdown gracefully
if ts == -1:
return True
# Timestamp should be less than 5 minutes, which indicates
# UDF wrote something atleast once in the last 5 minutes
timegap = time.time() - (ts / 1000)
if timegap > 300:
logger.info(
"Time gap is %d seconds, last line in the timestamp file was '%d'"
% (timegap, ts))
return False
except ValueError:
logger.info("Error parsing last line in the timestamp file => '" +
lines[-1] + "'")
return False
return True
@classmethod
def background_job_when_master_upgrade(cls):
'''
Start a background application which runs while component master service gets upgraded
:return:
'''
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][BGJob] Background Job test setup when upgrading Hive started"
)
logger.info("Creating hive tables for short background jobs")
query = "drop table if exists shortlr_hive_verify;\n"
query += "create table shortlr_hive_verify (userid string, age int);\n"
query += "drop table if exists shortlr_bline_verify;\n"
query += "create table shortlr_bline_verify (userid string, age int);\n"
query += "drop table if exists shortlr_bline_verify;\n"
query += "create table shortlr_bline_verify (userid string, age int);\n"
short_bgjob_setupfile = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
'shortlrsetup.sql')
util.writeToFile(query, short_bgjob_setupfile)
exit_code, stdout = Hive.run("-f " + short_bgjob_setupfile)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Background Job test setup when Hive upgrades failed due to exitcode = %d"
% exit_code)
logger.info("Running the Background Job when upgrading Hive")
UpgradePerNode.reportProgress(
"[INFO][Hive][BGJob] Long running job for Hive component upgrades started"
)
setqueue = ""
if Hive.isTezEnabled():
setqueue = "set tez.queue.name=%s; " % cls._yarn_queue
else:
setqueue = "set mapred.job.queue.name=%s; " % cls._yarn_queue
logger.info("**** Running Hive CLI Test ****")
query = setqueue + " insert overwrite table shortlr_hive_verify select userid, avg(age) from %s group by userid order by userid;" % cls._bgjtest_tbl
cls._shortbgj_hive_process = Hive.runQuery(query, background=True)
# Sleeping for 10 seconds to make sure that query initializes before Metastore is restarted
time.sleep(10)
logger.info("**** Running Beeline CLI Test ****")
query = setqueue + "\ninsert overwrite table shortlr_bline_verify select userid, avg(age) from %s group by userid order by userid;" % cls._bgjtest_tbl
cls._shortbgj_bline_process = Hive.runQueryOnBeeline(query,
readFromFile=True,
background=True)
UpgradePerNode.reportProgress(
"[INFO][Hive][BGJob] Background Job test setup when Hive upgrades finished"
)
@classmethod
def background_job_teardown_when_master_upgrade(cls):
'''
Clean up for background job which started before upgrading master services
:return:
'''
cls._shortbgj_hive_process = None
cls._shortbgj_bline_process = None
@classmethod
def verify_background_job_when_master_upgrade(cls):
'''
Validate background job Succeeded when master got upgraded
:return:
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
logger.info("Verify the successful completion of Hive CLI job")
starttime = time.time()
while (starttime - time.time() <
300) and cls._shortbgj_hive_process.poll() is None:
time.sleep(5)
procpoll = cls._shortbgj_hive_process.poll()
if procpoll is None:
logger.info("Killing Hive CLI process '%d'" %
cls._shortbgj_hive_process.pid)
Machine.killProcess(cls._shortbgj_hive_process.pid)
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Background job during master upgrade failed as Hive CLI failed to finish"
)
elif procpoll != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Background job during master upgrade failed as Hive CLI exited with '%d'"
% procpoll)
else:
UpgradePerNode.reportProgress(
"[PASSED][Hive][BGJob] Background job for Hive CLI during master upgrade finished successfully"
)
logger.info("Verify the successful completion of Beeline CLI job")
starttime = time.time()
while (starttime - time.time() <
120) and cls._shortbgj_bline_process.poll() is None:
time.sleep(5)
procpoll = cls._shortbgj_bline_process.poll()
if procpoll is None:
logger.info("Killing Beeline CLI process '%d'" %
cls._shortbgj_bline_process.pid)
Machine.killProcess(cls._shortbgj_bline_process.pid)
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Background job during master upgrade failed as Beeline CLI failed to finish"
)
elif procpoll != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][BGJob] Background job during master upgrade failed as Beeline CLI exited with '%d'"
% procpoll)
else:
UpgradePerNode.reportProgress(
"[PASSED][Hive][BGJob] Background job for Beeline CLI during master upgrade finished successfully"
)
@classmethod
def upgrade_master(cls, version, config=None):
'''
Upgrades Master services:
:param version: Version to be upgraded to
:param config: Config location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][Upgrade] Hive master components upgrade started")
cls.background_job_when_master_upgrade()
cls.switch_master_version('upgrade', version, config=config)
cls.verify_background_job_when_master_upgrade()
cls.background_job_teardown_when_master_upgrade()
UpgradePerNode.reportProgress(
"[INFO][Hive][Upgrade] Hive master components upgrade finished")
@classmethod
def upgrade_slave(cls, version, node, config=None):
'''
Upgrades slave services :
:param version: Version to be upgraded to
:param node: Slave Node
:param config: Config location
:return:
'''
logger.info("Hive does not have any slaves, so no slaves to upgrade")
@classmethod
def downgrade_master(cls, version, config=None):
'''
Downgrade Hive Master services
:param version: Version to be downgraded to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][Downgrade] Hive master components downgrade started")
cls.background_job_when_master_upgrade()
cls.switch_master_version('downgrade', version, config=config)
cls.verify_background_job_when_master_upgrade()
cls.background_job_teardown_when_master_upgrade()
UpgradePerNode.reportProgress(
"[INFO][Hive][Downgrade] Hive master components downgrade finished"
)
@classmethod
def downgrade_slave(cls, version, node, config=None):
'''
Downgrade Hive slave services
:param version: version to be downgraded to
:param config: Configuration location
'''
logger.info("Hive does not have any slaves, so no slaves to downgrade")
@classmethod
def switch_master_version(cls, action, version, config=None):
'''
Switches Hive master services' version
:param action: Whether to "upgrade" or "downgrade"
:param version: Version to be switched to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from beaver.component.hive import Hive
currentHiveVersion = Hive.getVersion()
if action == 'upgrade':
# Backup the database used by the Hive Metastore
logger.info(
"Performing backup of the Hive Metastore DB before starting the upgrade"
)
Hive.backupMetastoreDB(cls._metastore_backup_file)
node = Hive.getHiveHost()
# Stop the old Hive Metastore
logger.info("Stopping the Hive Metastore")
Hive.stopService(services=["metastore"])
# Upgrade Hive Metastore servers to new version
hdpSelect.changeVersion("hive-metastore", version, node)
if action == 'upgrade':
logger.info("Upgrading the Hive metastore schema")
Hive.upgradeSchema()
# Restart Hive Metastore servers one at a time
logger.info("Restarting the Hive Metastore")
Hive.startService(services=["metastore"])
# Start new Hive Server 2 instance
confHS2Port = Hive.getHiveserver2ThriftPort()
hs2port = util.getNextAvailablePort(node, confHS2Port)
hdpSelect.changeVersion("hive-server2", version, node)
Hive.modifyConfig(config,
services=['hiveserver2'],
restartService=False)
logger.info(
"Starting a new HiveServer2 at port '%d' for assisting rolling-upgrade"
% hs2port)
if hs2port != confHS2Port:
changes = {'hive-site.xml': {'hive.server2.thrift.port': hs2port}}
Hive.modifyConfig(changes,
services=["hiveserver2"],
restartService=False)
Hive.startService(services=["hiveserver2"])
cls._hs2_live_ports = [Hive.getHiveserver2ThriftPort(), hs2port]
# Deregister the old Hive Server 2 instances
logger.info("Deregistering the HiveServer2 on version '%s'" %
currentHiveVersion)
Hive.deregisterHiveServer2(version=currentHiveVersion)
from beaver.component.hcatalog import Hcatalog
# Stop the old WebHCat server
logger.info("Stopping the WebHCat server")
node = Config.get('templeton',
'TEMPLETON_HOST',
default=Machine.getfqdn())
webhcatPort = Config.get('templeton',
'TEMPLETON_PORT',
default="50111")
# Stop the old WebHCat server
logger.info("Stop the WebHCat server")
Hcatalog.stop(node)
# Upgrade WebHCat to the new version
hdpSelect.changeVersion("hive-webhcat", version, node)
# Start the WebHCat server
logger.info("Restarting the WebHCat server")
newConfDir = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
'localWebhcatConf')
if os.path.exists(newConfDir):
Hcatalog.start(node, hcat_confdir=newConfDir)
else:
Hcatalog.start(node)
@classmethod
def run_client_smoketest(cls, config=None, env=None):
'''
Run Smoke test after upgrading Client
:param config: Configuration location
:param env: Set Environment variables
'''
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][Smoke] Smoke test for Hive component started")
setqueue = ""
if Hive.isTezEnabled():
setqueue = "set tez.queue.name=%s; " % cls._yarn_queue
else:
setqueue = "set mapred.job.queue.name=%s; " % cls._yarn_queue
logger.info("**** Running Hive CLI Test ****")
query = setqueue + " insert overwrite table %s_hive_verify select userid, avg(age) from %s group by userid order by userid; " % (
cls._smoketest_tbl, cls._smoketest_tbl)
query += "select count(*) from %s_hive_verify;" % cls._smoketest_tbl
exit_code, stdout, stderr = Hive.runQuery(query,
stderr_as_stdout=False)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for Hive Metastore failed with exit code '%d'"
% exit_code)
logger.error(
"Smoke test for Hive failed with the following error: " +
stderr)
elif stdout.find("%d" % cls._num_of_rows_smoke) == -1:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for Hive Metastore failed to verify number of rows in output"
)
logger.error(
"Smoke test for Hive failed to find [%d] in output [%s]" %
(cls._num_of_rows_smoke, stdout))
else:
UpgradePerNode.reportProgress(
"[PASSED][Hive][Smoke] Smoke test for Hive Metastore succeeded"
)
logger.info("Smoke test for Hive Metastore succeeded")
logger.info("**** Running Beeline CLI Test ****")
query = setqueue + "\ndrop table if exists %s_bline_verify;\n" % cls._smoketest_tbl
query += "create table %s_bline_verify (userid string, age int);\n" % cls._smoketest_tbl
query += "insert overwrite table %s_bline_verify select userid, avg(age) from %s group by userid order by userid;\n" % (
cls._smoketest_tbl, cls._smoketest_tbl)
query += "select count(*) from %s_bline_verify;\n" % cls._smoketest_tbl
exit_code, stdout, stderr = Hive.runQueryOnBeeline(query,
readFromFile=True)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for HiveServer2 failed with exit code '%d'"
% exit_code)
logger.error(
"Smoke test for HiveServer2 failed with the following error: "
+ stderr)
elif stdout.find("%d" % cls._num_of_rows_smoke) == -1:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for HiveServer2 failed to verify number of rows in output"
)
logger.error(
"Smoke test for HiveServer2 failed to find [%d] in output [%s]"
% (cls._num_of_rows_smoke, stdout))
else:
logger.info("Smoke test for HiveServer2 succeeded")
logger.info("**** Running WebHCat Smoke Test ****")
query = "show tables;"
webhcatHost = Config.get('templeton',
'TEMPLETON_HOST',
default=Machine.getfqdn())
webhcatPort = Config.get('templeton',
'TEMPLETON_PORT',
default="50111")
url = "http://%s:%s/templeton/v1/ddl" % (webhcatHost, webhcatPort)
params = {'exec': query}
status_code, stdout = util.curl(url, method='POST', params=params)
if status_code != 200:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for WebHCat failed due to status code = %d"
% status_code)
else:
logger.info("Smoke test for WebHCat succeeded")
UpgradePerNode.reportProgress(
"[INFO][Hive][Smoke] Smoke test for Hive component finished")
@classmethod
def testAfterAllSlavesRestarted(cls):
'''
Function to test upgrade is done properly after all master and slaves are upgraded for Hdfs, yarn and Hbase
:return:
'''
logger.info("Hive does not have any slaves, so nothing to do here")
def downgrade_master(cls, version, config=None):
'''
Downgrade HDFS Master services
:param version: Version to be downgraded to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
if HDFS.isHAEnabled():
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] HA Namenode Downgrade Started ")
nodes = []
nodes.append(HDFS.getNamenodeByState('standby'))
nodes.append(HDFS.getNamenodeByState('active'))
for node in nodes:
HDFS.resetNamenode('stop', host=node)
HDFS.resetZkfc('stop', hosts=node.split())
# BUG-25534: There is no package for zkfc. So just updating the NN is enough.
hdpSelect.changeVersion("hadoop-hdfs-namenode", version, node)
HDFS.resetNamenode('start', config=config, host=node)
HDFS.resetZkfc('start', hosts=node.split())
# lets make sure the NN is out of safemode before we proceed to the next namenode
HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' %
node)
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] HA Namenode Downgrade Finished ")
jn_nodes = HDFS.getJournalNodes()
if len(jn_nodes) < 3:
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] Less than three Journal Nodes. Not going to do Downgrade "
)
return
#Loop through all the JNs and stop flip start one at a time
hdfs_user = Config.get('hadoop', 'HDFS_USER')
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] HA Journal Node Upgrade Started")
cls.ensure_all_jns_are_up(jn_nodes)
#Loop through all the JNs and stop flip start one at a time
for node in jn_nodes:
Hadoop.resetService(hdfs_user,
node,
"journalnode",
'stop',
binFolder="sbin")
hdpSelect.changeVersion("hadoop-hdfs-journalnode", version,
node)
Hadoop.resetService(hdfs_user,
node,
"journalnode",
'start',
binFolder="sbin")
time.sleep(5)
cls.ensure_all_jns_are_up(jn_nodes)
cls.ensure_nn_is_active()
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] HA Journal Node Downgrade Finished ")
else:
## TODO add code to upgrade SNN
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] Namenode Downgrade Started ")
node = HDFS.getNamenode()
HDFS.stopNamenode()
hdpSelect.changeVersion("hadoop-hdfs-namenode", version, node)
HDFS.startNamenode(config=config)
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] Namenode Downgrade Finished ")
# upgrade SNN
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade]Secondary Namenode Downgrade Started ")
node = HDFS.getSecondaryNamenode()
HDFS.stopSecondaryNamenode()
hdpSelect.changeVersion("hadoop-hdfs-secondarynamenode", version,
node)
HDFS.startSecondaryNamenode(config=config)
UpgradePerNode.reportProgress(
"[INFO][HDFS][Downgrade] Secondary Namenode Downgrade Finished"
)
xmlDict["yarn.scheduler.capacity.root.%s.acl_administer_jobs" %
component] = "*"
util.dumpTextString(xmlDict, "====== PLANNED QUEUES ======",
"==================")
master_capacity_file = os.path.join(Config.getEnv("WORKSPACE"),
"tests", "rolling_upgrade", "yarn",
"data", "capacity-scheduler.xml")
modified_capacity_file = os.path.join(Config.getEnv("ARTIFACTS_DIR"),
"capacity-scheduler.xml")
Machine.copy(master_capacity_file, modified_capacity_file)
util.writePropertiesToConfigXMLFile(modified_capacity_file,
modified_capacity_file, xmlDict)
#util.dumpText(modified_capacity_file, "====== capacity-scheduler.xml ======", "==================")
if RuSetup._defaultQueue:
components.remove("default")
return modified_capacity_file
__localTestDir = os.path.join(Config.getEnv("WORKSPACE"), "tests",
"rolling_upgrade")
__confFile = __localTestDir + '/conf.json'
__depFile = __localTestDir + '/dep.json'
# Public: all components to run the test
COMPONENTS_TO_TEST = RuSetup.getComponnetsToTest(__confFile, __depFile)
# Public: all components to flip binary
COMPONENTS_TO_FLIP = RuSetup.getComponentsToFlip(__confFile, __depFile)
# Public: components that are affected by the tested components
COMPONENTS_AFFECTED = RuSetup.getComponentsAffected(__confFile, __depFile)
# Public: components that we should import
COMPONENTS_TO_IMPORT = list(set(COMPONENTS_TO_TEST) | set(COMPONENTS_TO_FLIP))
class ruFalcon:
_base_falcon_dir = None
_local_workspace = Config.getEnv("WORKSPACE")
_job_user = Config.getEnv("USER")
_falcon_user = Config.get("falcon", "FALCON_USER")
_job_start_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%MZ")
@classmethod
def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Upload Data to HDFS before Upgrade starts
Creates /user/hrt_qa/falcon/ dir on HDFS
Upload demo files to /user/hrt_qa/falcon
'''
logger.info("Falcon - starting background job setup")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][BGJobSetup] starting Falcon background job setup")
from beaver.component.hadoop import HDFS
if not cls._base_falcon_dir:
cls._base_falcon_dir = '/user/%s/falcon' % cls._job_user
exit_code, stdout = HDFS.createDirectory(cls._base_falcon_dir,
user=cls._job_user,
perm=777,
force=True)
ruAssert("Falcon", exit_code == 0,
'[BGJobSetup] could not create dir on hdfs.')
HDFS.copyFromLocal(os.path.join(cls._local_workspace, "tests",
"rolling_upgrade", "falcon", "demo"),
cls._base_falcon_dir,
user=cls._job_user)
## Create dirs for falcon clusters
exit_code, stdout = HDFS.createDirectory(
"/apps/falcon/primaryCluster/staging",
user=cls._falcon_user,
perm=777,
force=True)
ruAssert("Falcon", exit_code == 0,
'[BGJobSetup] could not create staging dir on hdfs ')
exit_code, stdout = HDFS.createDirectory(
"/apps/falcon/primaryCluster/working",
user=cls._falcon_user,
perm=755,
force=True)
ruAssert("Falcon", exit_code == 0,
'[BGJobSetup] could not create dir on hdfs.')
exit_code, stdout = HDFS.createDirectory(
"/apps/falcon/backupCluster/staging",
user=cls._falcon_user,
perm=777,
force=True)
ruAssert("Falcon", exit_code == 0,
'[BGJobSetup] could not create dir on hdfs.')
exit_code, stdout = HDFS.createDirectory(
"/apps/falcon/backupCluster/working",
user=cls._falcon_user,
perm=755,
force=True)
ruAssert("Falcon", exit_code == 0,
'[BGJobSetup] could not create dir on hdfs.')
## Create cluster entities.
cls.createClusterEntities("USWestOregon", "oregonHadoopCluster",
"primaryCluster")
cls.createClusterEntities("USEastVirginia", "virginiaHadoopCluster",
"backupCluster")
if runSmokeTestSetup:
logger.info("**** Running Falcon Smoke Test Setup ****")
cls.smoke_test_setup()
logger.info("Falcon - completed background job setup")
return
@classmethod
def smoke_test_setup(cls):
'''
Setup required to run Smoke test
'''
logger.info(
"Falcon smoke test setup is same as background job setup. Nothing to do"
)
return
@classmethod
def run_background_job(cls, runSmokeTestSetup=True, config=None):
'''
Runs background long running Yarn Job
:param runSmokeTestSetup: Runs smoke test setup if set to true
:param config: expected configuration location
:return: Total number of long running jobs started
'''
logger.info("Falcon - start running background job")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[PASSED][Falcon][BGJob] starting Falcon background jobs")
BACKGROUND_JOB_DIR = os.path.join(cls._local_workspace, "tests",
"rolling_upgrade", "falcon",
"falconChurnDemo")
# Add clusters
clusterXml = os.path.join(cls._local_workspace, "primaryCluster.xml")
logger.info("Content of cluster xml %s" % clusterXml)
f = open(clusterXml, 'r')
logger.info(f.read())
f.close()
CMD = " entity -type cluster -submit -file " + clusterXml
cls.runCommand(cls._job_user, CMD)
clusterXml = os.path.join(cls._local_workspace, "backupCluster.xml")
CMD = " entity -type cluster -submit -file " + clusterXml
cls.runCommand(cls._job_user, CMD)
# Add feed
feedXml = os.path.join(BACKGROUND_JOB_DIR, "rawEmailFeed.xml")
logger.info("Content of feed xml %s" % feedXml)
f = open(feedXml, 'r')
logger.info(f.read())
f.close()
CMD = " entity -type feed -submit -file " + feedXml
cls.runCommand(cls._job_user, CMD)
# Add Process
processXml = os.path.join(BACKGROUND_JOB_DIR, "emailIngestProcess.xml")
logger.info("Content of process xml %s" % processXml)
f = open(processXml, 'r')
logger.info(f.read())
f.close()
CMD = " entity -type process -submit -file " + processXml
cls.runCommand(cls._job_user, CMD)
# Schedule Feed and Process
CMD = " entity -type feed -schedule -name rawEmailFeed"
cls.runCommand(cls._job_user, CMD)
CMD = " entity -type process -schedule -name rawEmailIngestProcess"
cls.runCommand(cls._job_user, CMD)
cls._job_start_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%MZ")
logger.info("Falcon - completed running background job")
return 1
@classmethod
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run smoke test for yarn
:param smoketestnumber: Used for unique output log location
'''
logger.info("Falcon - start running smoke test")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][Smoke] starting Falcon smoke test")
cls.runCommand(cls._falcon_user, "admin -version")
# QE-4347: Do not run the list call as smoke is run before long running job is called.
# cls.runCommand(cls._job_user,
# " instance -type process -name rawEmailIngestProcess -list ")
logger.info("Falcon - finished running smoke test")
UpgradePerNode.reportProgress(
"[INFO][Falcon][Smoke] finishing Falcon smoke test")
return
@classmethod
def stopFalconLongRunningJob(cls):
logger.info("Falcon - Begin function stopFalconLongRunningJob")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][BGJobStop]Stopped/deleting Falcon background jobs started"
)
CMD = " instance -type process -name rawEmailIngestProcess " \
" -list -start 2016-07-25T00:00Z -end 2018-07-25T00:00Z " \
" -filterBy STARTEDAFTER:" + cls._job_start_time + " -numResults 1000"
exitcode, output = cls.runCommand(cls._job_user, CMD)
success_count = output.count("SUCCEEDED")
fail_count = output.count("KILLED")
ruAssert(
"Falcon", success_count > 0 and fail_count == 0,
"[BGJobCheck] Long running job failed with " + str(fail_count) +
" FAILED instances")
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
# Delete entities
cls.runCommand(
cls._job_user,
" entity -type process -delete -name rawEmailIngestProcess")
cls.runCommand(cls._job_user,
" entity -type feed -delete -name rawEmailFeed")
cls.runCommand(cls._job_user,
" entity -type cluster -delete -name primaryCluster")
cls.runCommand(cls._job_user,
" entity -type cluster -delete -name backupCluster")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][BGJobStop]Stopped/deleting Falcon background jobs")
logger.info("Falcon - End function stopFalconLongRunningJob")
return
@classmethod
def background_job_teardown(cls):
'''
Cleanup for long running Falcon job
'''
logger.info(
"Tear down happened when long running job is stopped. Nothing to do"
)
return
@classmethod
def falcon_teardown(cls):
'''
Clean up entities from Falcon
'''
# Delete entities
logger.info("Falcon - Begin function falcon_teardown")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][BGJobTeardown] Falcon background jobs cleanup started"
)
cls.runCommand(
cls._job_user,
" entity -type process -delete -name rawEmailIngestProcess",
ignoreError=True)
cls.runCommand(cls._job_user,
" entity -type feed -delete -name rawEmailFeed",
ignoreError=True)
cls.runCommand(cls._job_user,
" entity -type cluster -delete -name primaryCluster",
ignoreError=True)
cls.runCommand(cls._job_user,
" entity -type cluster -delete -name backupCluster",
ignoreError=True)
logger.info("Falcon - End function falcon_teardown")
@classmethod
def verifyLongRunningJob(cls):
'''
Validate long running background job after end of all component upgrade
'''
logger.info("Verified when stopping Long Running Job ")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][BGJobCheck] Verify Falcon Background Application")
cls.runCommand(cls._falcon_user, "admin -version")
@classmethod
def background_job_when_master_upgrade(cls):
'''
Start a background application which runs while component master service gets upgraded
:return:
'''
logger.info("TODO - short backgroundjob when master upgrade/downgrade")
@classmethod
def background_job_teardown_when_master_upgrade(cls):
'''
Clean up for background job which started before upgrading master services
:return:
'''
logger.info(
"TODO - short backgroundjob teardown when master upgrade/downgrade"
)
@classmethod
def verify_background_job_when_master_upgrade(cls):
'''
Validate background job Succeeded when master got upgraded
:return:
'''
logger.info(
"Falcon - Begin function verify_background_job_when_master_upgrade"
)
cls.verifyLongRunningJob()
@classmethod
def upgrade_master(cls, version, config=None):
'''
Upgrades Master services:
:param version: Version to be upgraded to
:param config: Config location
'''
logger.info("Falcon - Begin function upgrade_master")
cls.background_job_when_master_upgrade()
cls.change_master_version(version, config)
cls.verify_background_job_when_master_upgrade()
cls.background_job_teardown_when_master_upgrade()
@classmethod
def upgrade_slave(cls, version, node, config=None):
'''
Upgrades slave services :
:param version: Version to be upgraded to
:param node: Slave Node
:param config: Config location
:return:
'''
logger.info("Falcon has only a master, No slave to upgrade.")
@classmethod
def downgrade_master(cls, version, config=None):
'''
Exactly same as upgrade master. There is no difference in steps.
:param version: Version to be downgraded to
:param config: Config location
:return:
'''
logger.info("Falcon - Begin function downgrade_master ")
cls.change_master_version(version, config, logText="Downgrade")
@classmethod
def change_master_version(cls, version, config=None, logText="Upgrade"):
'''
Changes Master services:
:param version: Version to be changed to
:param config: Config location
'''
logger.info("Falcon - Begin function change_master_version ")
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from time import sleep
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
node = Falcon.get_falcon_server()
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][%s] Falcon server for node %s to version %s started"
% (logText, node, version))
Falcon.stop(cls._falcon_user, node)
sleep(30)
hdpSelect.changeVersion("falcon-server", version, node)
hdpSelect.changeVersion("falcon-client", version, node)
Falcon.start(cls._falcon_user, node)
sleep(60)
UpgradePerNode.reportProgress(
"[INFO][Falcon][%s] Falcon server for node %s to version %s finished"
% (logText, node, version))
return
@classmethod
def downgrade_slave(cls, version, node, config=None):
'''
Downgrade slave services
:param version: version to be downgraded to
:param config: Configuration location
'''
logger.info("Falcon runs in standalone mode. Nothing to do.")
@classmethod
def run_client_smoketest(cls, config=None, env=None):
'''
Run Smoke test after upgrading Client
:param config: Configuration location
:param env: Set Environment variables
'''
logger.info("Falcon - Begin function run_client_smoketest ")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][Smoke] starting Falcon client smoke test")
cls.run_smoke_test("123", config=None)
@classmethod
def testAfterAllSlavesRestarted(cls):
'''
Function to test upgrade is done properly after all master and slaves are upgraded for Hdfs, yarn and Hbase
:return:
'''
logger.info("Falcon - Begin function testAfterAllSlavesRestarted ")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][AFTER SLAVE PASSED] starting Falcon smoke test")
cls.runCommand(cls._falcon_user, "admin -version")
cls.runCommand(
cls._job_user,
" instance -type process -name rawEmailIngestProcess -list -orderBy startTime"
)
@classmethod
def runCommand(cls, user, cmd, ignoreError=True):
"""
Function to run falcon commands cmd as a certain user
:param user:
:param cmd:
:return:
"""
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return 0, ""
exit_code, output = Falcon.runas(user, cmd)
if ignoreError:
if exit_code != 0:
#logger.info("Warning (ignoreError=True): Non-zero exit code when running command " + cmd + " as user " + user)
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[FAILED][Falcon] Warning (ignoreError=True): Non-zero exit code when running command %s as user %s "
% (cmd, user))
else:
ruAssert(
"Falcon", exit_code == 0,
"[RunCommand] Could not run command " + cmd + " as user " +
user)
return exit_code, output
@classmethod
def createClusterEntities(cls, colo, desc, name):
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
from beaver.component.hadoop import Hadoop, HDFS, YARN
write_endpoint = Hadoop.getFSDefaultValue()
webhdfs_scheme = 'webhdfs'
if HDFS.isHttpsEnabled():
webhdfs_scheme = 'swebhdfs'
read_endpoint = '%s://%s:%s' % (
webhdfs_scheme, write_endpoint.split('/')[2].split(':')[0],
HDFS.getNNWebPort())
execute_endpoint = YARN.getResourceManager()
falconNode = Falcon.get_falcon_server()
from beaver.component.oozie import Oozie
oozieUrl = Oozie.getOozieUrl()
entityText = "<?xml version=\"1.0\"?>" \
"<cluster colo=\"" + colo + "\" description=\"" + desc + "\" name=\"" + name + "\" " \
"xmlns=\"uri:falcon:cluster:0.1\"> " \
"<interfaces> " \
"<interface type=\"readonly\" endpoint=\""+read_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"write\" endpoint=\""+write_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"execute\" endpoint=\"" + execute_endpoint + "\" version=\"0.20.2\"/> " \
"<interface type=\"workflow\" endpoint=\"" + oozieUrl + "\" version=\"3.1\"/>" \
"<interface type=\"messaging\" endpoint=\"" \
"tcp://" + falconNode + ":61616?daemon=true\" version=\"5.1.6\"/>" \
"</interfaces>" \
"<locations>" \
"<location name=\"staging\" path=\"/apps/falcon/" + name + "/staging\" />" \
"<location name=\"temp\" path=\"/tmp\" />" \
"<location name=\"working\" path=\"/apps/falcon/" + name + "/working\" />" \
"</locations>" \
"<ACL owner=\"" + cls._job_user + "\" group=\"users\" permission=\"0755\"/>"
if Hadoop.isSecure():
realm = HDFS.getConfigValue(
'dfs.namenode.kerberos.principal').split('@')[1]
entityText += "<properties> <property name=\"dfs.namenode.kerberos.principal\" value=\"nn/_HOST@" + realm + "\"/> </properties>"
entityText += "</cluster>"
textFile = open(os.path.join(cls._local_workspace, name + ".xml"), "w")
textFile.write("%s" % entityText)
textFile.close()
return
def getHadoopConfDir(cls):
return Config.get('hadoop', 'HADOOP_CONF')
def run_client_smoketest(cls, config=None, env=None):
'''
Run Smoke test after upgrading Client
:param config: Configuration location
:param env: Set Environment variables
'''
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Hive][Smoke] Smoke test for Hive component started")
setqueue = ""
if Hive.isTezEnabled():
setqueue = "set tez.queue.name=%s; " % cls._yarn_queue
else:
setqueue = "set mapred.job.queue.name=%s; " % cls._yarn_queue
logger.info("**** Running Hive CLI Test ****")
query = setqueue + " insert overwrite table %s_hive_verify select userid, avg(age) from %s group by userid order by userid; " % (
cls._smoketest_tbl, cls._smoketest_tbl)
query += "select count(*) from %s_hive_verify;" % cls._smoketest_tbl
exit_code, stdout, stderr = Hive.runQuery(query,
stderr_as_stdout=False)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for Hive Metastore failed with exit code '%d'"
% exit_code)
logger.error(
"Smoke test for Hive failed with the following error: " +
stderr)
elif stdout.find("%d" % cls._num_of_rows_smoke) == -1:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for Hive Metastore failed to verify number of rows in output"
)
logger.error(
"Smoke test for Hive failed to find [%d] in output [%s]" %
(cls._num_of_rows_smoke, stdout))
else:
UpgradePerNode.reportProgress(
"[PASSED][Hive][Smoke] Smoke test for Hive Metastore succeeded"
)
logger.info("Smoke test for Hive Metastore succeeded")
logger.info("**** Running Beeline CLI Test ****")
query = setqueue + "\ndrop table if exists %s_bline_verify;\n" % cls._smoketest_tbl
query += "create table %s_bline_verify (userid string, age int);\n" % cls._smoketest_tbl
query += "insert overwrite table %s_bline_verify select userid, avg(age) from %s group by userid order by userid;\n" % (
cls._smoketest_tbl, cls._smoketest_tbl)
query += "select count(*) from %s_bline_verify;\n" % cls._smoketest_tbl
exit_code, stdout, stderr = Hive.runQueryOnBeeline(query,
readFromFile=True)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for HiveServer2 failed with exit code '%d'"
% exit_code)
logger.error(
"Smoke test for HiveServer2 failed with the following error: "
+ stderr)
elif stdout.find("%d" % cls._num_of_rows_smoke) == -1:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for HiveServer2 failed to verify number of rows in output"
)
logger.error(
"Smoke test for HiveServer2 failed to find [%d] in output [%s]"
% (cls._num_of_rows_smoke, stdout))
else:
logger.info("Smoke test for HiveServer2 succeeded")
logger.info("**** Running WebHCat Smoke Test ****")
query = "show tables;"
webhcatHost = Config.get('templeton',
'TEMPLETON_HOST',
default=Machine.getfqdn())
webhcatPort = Config.get('templeton',
'TEMPLETON_PORT',
default="50111")
url = "http://%s:%s/templeton/v1/ddl" % (webhcatHost, webhcatPort)
params = {'exec': query}
status_code, stdout = util.curl(url, method='POST', params=params)
if status_code != 200:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Smoke] Smoke test for WebHCat failed due to status code = %d"
% status_code)
else:
logger.info("Smoke test for WebHCat succeeded")
UpgradePerNode.reportProgress(
"[INFO][Hive][Smoke] Smoke test for Hive component finished")
def getMapredUser(cls):
return Config.get('hadoop', 'MAPRED_USER')
# company and Hortonworks, Inc, any use, reproduction, modification,
# redistribution or other exploitation of all or any part of the contents
# of this file is strictly prohibited.
#
#
import datetime
import logging
import time
from beaver.component.ambari import Ambari
from beaver.component.xa import Xa, XaPolicy
from beaver.config import Config
logger = logging.getLogger(__name__)
ARTIFACTS_DIR = Config.getEnv('ARTIFACTS_DIR')
_ambari_host_1 = Config.get("multicluster", "AMBARI_GATEWAY1")
_ambari_host_2 = Config.get("multicluster", "AMBARI_GATEWAY2")
source_weburl = Ambari.getWebUrl(hostname=_ambari_host_1)
target_weburl = Ambari.getWebUrl(hostname=_ambari_host_2)
source_datacenter = target_datacenter = "default"
primaryCluster = source_datacenter + "$" + Ambari.getClusterName(
weburl=source_weburl)
backupCluster = target_datacenter + "$" + Ambari.getClusterName(
weburl=target_weburl)
policy_prefix = 'pol_for_'
class BeaconRanger:
def __init__(self):
def runAgent(cls, name, conffile, user=None, cwd=None, env=None):
flume_conf = Config.get('flume-ng', 'FLUME_CONF')
cmd = "agent -n %s -c %s -f %s" % (name, flume_conf, conffile)
return cls.runInBackgroundAs(user, cmd, cwd=cwd, env=env)
def createRangerHivePolicy(cls,
database,
udf=None,
table=None,
column=None,
userList=None,
weburl=None):
BeaconRanger.changePolicyInterval()
logger.info('creating policy for %s' % database)
users = [Config.getEnv('USER')] if userList is None else userList
if table is not None:
polItem = XaPolicy.createPolicyItem(
userList=users, PermissionList=XaPolicy.HIVE_ALL_PERMISSIONS)
if column is not None:
policy = XaPolicy.getPolicyJson(policy_prefix + '_' +
database + '_table_' + table +
'_col_' + column,
'hive', [polItem],
ambariWeburl=weburl,
database=database,
table=table,
column=column)
else:
policy = XaPolicy.getPolicyJson(policy_prefix + '_' +
database + '_table_' + table,
'hive', [polItem],
ambariWeburl=weburl,
database=database,
table=table,
column='*')
elif udf is not None:
polItem = XaPolicy.createPolicyItem(
userList=users, PermissionList=XaPolicy.HIVE_ALL_PERMISSIONS)
policy = XaPolicy.getPolicyJson(policy_prefix + '_' + database +
'_udf_' + udf,
'hive', [polItem],
ambariWeburl=weburl,
database=database,
udf=udf)
else:
# repladmin check
polItem = XaPolicy.createPolicyItem(
userList=users,
PermissionList=[
XaPolicy.PERMISSION_CREATE, XaPolicy.PERMISSION_SELECT,
XaPolicy.PERMISSION_DROP, XaPolicy.PERMISSION_REPL_ADMIN
])
policy = XaPolicy.getPolicyJson(policy_prefix + '_' + database,
'hive', [polItem],
ambariWeburl=weburl,
database=database,
table='*',
column='*')
logger.info('going to create policy: ' + str(policy))
status_code, response = Xa.createPolicy_api_v2(
policy,
policyActivationWaitTime=BeaconRanger.policyActivationWaitTime,
weburl=weburl)
if status_code == 200 and weburl != target_weburl:
BeaconRanger.policiesAddedBeforeTest.append(policy)
def runAdminReport(cls):
exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -report")
return output
#
#
import os, re, string, time, socket, logging, platform, urllib2, collections, datetime, json
import urllib, sys
from beaver.component.hadoop import Hadoop, HDFS, MAPRED, YARN
from beaver.component import HadoopJobHelper
from beaver.machine import Machine
from beaver.config import Config
from beaver import util
from beaver import configUtils
import pytest
import random
logger = logging.getLogger(__name__)
HADOOPQA_USER = Config.get('hadoop', 'HADOOPQA_USER')
INPUT_CSV_FILE = 'inputCSV.csv'
TEST_FOLDER = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
'ru_phoenix_testFolder')
TEST_TABLE = 'testTable'
logger = logging.getLogger(__name__)
HOST = None
PHOENIX_TEST_TABLE = 'basicTable'
EXAMPLE_ROWS = [
'1,John,Snow,The Wall\n'
'2,Jaime,Lanninster,Kings Landing\n'
def runas(cls, user, cmd, cwd=None, env=None, logoutput=True):
flume_cmd = Config.get('flume-ng', 'FLUME_CMD')
flume_cmd += " " + cmd
return Machine.runas(user, flume_cmd, cwd=cwd, env=env, logoutput=logoutput)
def run(cls, cmd, logoutput=True):
return Machine.run(Config.get('hbase', 'HBASE_CMD') + " " + cmd, logoutput=logoutput)
def getDatanodes(cls):
slaveFile = os.path.join(Config.get('hadoop', 'HADOOP_CONF'),"slaves")
f = open(slaveFile, "r")
datanodes = f.readlines()
return datanodes
def getVersion(cls):
return Config.get('hadoop', 'VERSION')
def perform_express_upgrade(self):
COMPONENT = str(self.COMPONENT)
STACK_TYPE = str(self.STACK_TYPE)
env = {}
env['DISPLAY'] = self.DISPLAY
# Update pom.xml of uifrm with Markers - applies for tests that involve a combination of API and UI tests run within the same split
ambariutils.update_pom_xml_with_markers(self.LOCAL_WORK_DIR)
SRC_DIR = os.path.join(Config.getEnv('WORKSPACE'), 'apitestframework')
print "SRC_DIR = ", SRC_DIR
# Change the current directory for api test code
LOCAL_WORK_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), COMPONENT, 'apitestframework')
# Copy the ambari api test code to artifacts dir
shutil.copytree(SRC_DIR, LOCAL_WORK_DIR)
# Change the permission of openstack-keypairfile
os.chmod(os.path.join(LOCAL_WORK_DIR, 'src/resources/openstack-keypair'), 0400)
# populate the config properties file for api test code
configPropChanges = ambariutils.updateProperties()
# Change the current directory for api test code
LOCAL_WORK_DIR = os.path.join(
Config.getEnv('ARTIFACTS_DIR'), COMPONENT, 'apitestframework', 'src', 'resources'
)
util.writePropertiesToFile(
os.path.join(LOCAL_WORK_DIR, 'config.properties'), os.path.join(LOCAL_WORK_DIR, 'config.properties'),
configPropChanges
)
# Change the current directory for api test code
LOCAL_WORK_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), COMPONENT, 'apitestframework')
testSuiteLocation = "src/test/suites"
# Check what is the STACK_TYPE. Based on the stack type we will decide which test suite to select
logger.info(
"STACK TYPE is " + STACK_TYPE + ". So going into " + STACK_TYPE + " if block for test suite "
"selection"
)
if STACK_TYPE == "HDF":
# Setup for HDF EU. We can now add an else method to handle ru
if 'ambarieu-hdf-downgrade' in COMPONENT:
testSuiteFile = "ambarieu-hdf-downgrade.suite"
elif 'ambarieu-hdf' in COMPONENT:
testSuiteFile = "ambarieu-hdf.suite"
logger.info("Setting HDF testsuite file to " + testSuiteFile)
elif STACK_TYPE == "HDP" and "ambarieu-hdp-hdf" in COMPONENT:
testSuiteFile = "ambarieu-hdp-hdf.suite"
else:
if 'e2e' not in COMPONENT:
# These are system tests so use suite filename sent from json file
testSuiteFile = COMPONENT + ".suite"
else:
if 'dwngd' in COMPONENT: # For EU E2E downgrade runs
testSuiteFile = "ambari-expressupgrade-downgrade-e2e.suite"
elif 'mlevel' in COMPONENT: # For EU E2E multi-level paths runs
testSuiteFile = "ambari-expressupgrade-mlevel-e2e.suite"
elif 'denygpl' in COMPONENT: # For Deny GPL tests in EU E2E
testSuiteFile = "ambarieu-denygpl.suite"
elif 'wkflow' in COMPONENT: # For EU E2E workflow suite runs
testSuiteFile = "ambari-expressupgrade-wkflow-e2e.suite"
elif 'iopmigration' in COMPONENT: # Added for IOP migration E2E tests
testSuiteFile = "ambari-iopmigration-e2e.suite"
elif 'iopintg' in COMPONENT: # Added for IOP integration tests
testSuiteFile = "ambarieu-iopintg-e2e.suite"
elif 'patchupgradeintg-ru' in COMPONENT: # Added for RU PU integration tests
testSuiteFile = "ambarieu-patchupgradeintg-ru-e2e.suite"
elif 'patchupgradeintg-thirddigit' in COMPONENT: # Added for EU PU 3rd digit integration tests
testSuiteFile = "ambarieu-patchupgradeintg-thirddigit-e2e.suite"
elif 'patchupgradeintg-revert' in COMPONENT: # Added for EU PU integration tests with revert
testSuiteFile = "ambarieu-patchupgradeintg-revert-e2e.suite"
elif 'patchupgradeintg' in COMPONENT: # Added for EU PU integration tests
testSuiteFile = "ambarieu-patchupgradeintg-e2e.suite"
elif 'experiment' in COMPONENT: # Added for full EU integration tests
testSuiteFile = "ambari-expressupgrade-experiment-e2e.suite"
else: # Default ofr EU E2E runs
testSuiteFile = "ambari-expressupgrade-upgrade-e2e.suite"
logger.info("Opening test suite file : " + testSuiteFile + " for test execution")
file = open(os.path.join(LOCAL_WORK_DIR, testSuiteLocation, testSuiteFile))
testSuite = json.load(file)
file.close()
# magic word to use as key in suite file
magic = "split" + self.splitNumStr
print "magic key is : ", magic
if 'experiment' in COMPONENT or 'patchupgradeintg' in COMPONENT or 'iopintg' in COMPONENT:
magic = "split1"
print "magic key for experiment EU/PU run is : ", magic
# Update pom.xml for API framework with Markers info
ambariutils.update_pom_xml_with_markers(LOCAL_WORK_DIR)
upgrade_test_results = {}
# Iterate over the list of all test classes in the split and execute them
logger.info("=====Starting Express Upgrade tests=========")
if testSuite.has_key(magic):
for testName in testSuite[magic]:
if not ambariutils.isTestClassPresent(testName, LOCAL_WORK_DIR):
LOCAL_WORK_DIR = ambariutils.switchDirectory(LOCAL_WORK_DIR, COMPONENT)
logger.info('LOCAL_WORK_DIR %s ', LOCAL_WORK_DIR)
logger.info('================Running %s with maven===============' % (testName))
self.LOCAL_WORK_DIR = LOCAL_WORK_DIR
exit_code, stdout = self.Maven2runas(
' -Dtest=%s -DfailIfNoTests=false test' % testName, cwd=self.LOCAL_WORK_DIR, env=env, user='******'
)
UpgradeLogger.reportProgress('================Finished %s ========================' % (testName), True)
logger.info('Exit code of the test: %s ' % (exit_code))
if exit_code != 0:
upgrade_test_results.update(UpgradeLogger.get_stack_trace(testName, LOCAL_WORK_DIR))
UpgradeLogger.reportProgress("Test failure encountered: %s" % (testName), False)
# Do not run any further tests if Upgrade itself has failed
if self.is_upgrade_executed():
if not self.is_stack_upgrade_success():
UpgradeLogger.reportProgress('Express Upgrade failed, aborting rest of the tests', False)
break
else:
UpgradeLogger.reportProgress(
'Error(s) in steps before starting Upgrade, aborting rest of the tests', False
)
break
else:
print "================Not correct test suite format========================"
if len(upgrade_test_results) > 0:
UpgradeLogger.reportProgress("=====List of failed test(s)=====\n", False)
for key, value in upgrade_test_results.items():
UpgradeLogger.reportProgress(key, False)
UpgradeLogger.reportProgress("=====Error details for failed test(s)=====", False)
for key, value in upgrade_test_results.items():
UpgradeLogger.reportProgress("Test:%s AND Failure details:\n %s" % (key, value), False)
UpgradeLogger.reportProgress("=======================================================", True)
else:
UpgradeLogger.reportProgress("=====Express Upgrade test(s) completed successfully=========", True)
# Gather reports for tests executed from apitestframework dir
LOCAL_WORK_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), COMPONENT, 'apitestframework')
Machine.runas('root', 'chmod -R 777 ' + LOCAL_WORK_DIR)
uifrmReportDirectory = os.path.join(LOCAL_WORK_DIR, '..', 'target/surefire-reports')
if not os.path.exists(uifrmReportDirectory):
Machine.runas('root', 'mkdir -p ' + uifrmReportDirectory)
Machine.runas('root', 'chmod -R 777 ' + uifrmReportDirectory)
logger.info('Created path for reporting')
Machine.runas('root', 'chmod -R 777 ' + os.path.join(LOCAL_WORK_DIR, '..', 'target'))
apiReportDirectory = os.path.join(LOCAL_WORK_DIR, 'target', 'surefire-reports', 'junitreports')
if os.path.exists(apiReportDirectory):
files = os.listdir(os.path.join(LOCAL_WORK_DIR, 'target', 'surefire-reports', 'junitreports'))
for file in files:
shutil.copy(
os.path.join(LOCAL_WORK_DIR, 'target', 'surefire-reports', 'junitreports', file),
os.path.join(LOCAL_WORK_DIR, '..', 'target', 'surefire-reports')
)
# Switch back to uifrm dir for reporting purposes
LOCAL_WORK_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), COMPONENT)
self.LOCAL_WORK_DIR = LOCAL_WORK_DIR
def getDruidHome(cls):
'''
Returns Druid_HOME
'''
return Config.get('druid', 'DRUID_HOME')
def verifySchemaFunctionality(cls):
'''
We verify that the system can operate with SCHEMA functionality.
'''
from beaver.component.phoenix import Phoenix
from beaver.component.hbase import HBase
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
HBASE_HOME = Config.get('hbase', 'HBASE_HOME')
HBASE_CONF_DIR = os.path.join(HBASE_HOME, "conf")
#We verify the schema functionality.
HBASE_CHANGES = {}
HBASE_CHANGES['hbase-site.xml'] = {}
HBASE_CHANGES['hbase-site.xml'][
'phoenix.schema.isNamespaceMappingEnabled'] = 'true'
HBASE_CHANGES['hbase-site.xml'][
'phoenix.schema.mapSystemTablesToNamespace'] = 'true'
TEST_TABLE_A = 'Table_A'
SCHEMA_1 = 'SCHEMA_1'
masterNodes = HBase.getAllMasterNodes()
regionNodes = HBase.getRegionServers()
hbase_allnodes = masterNodes + regionNodes
gateway_node = Machine.getfqdn()
if gateway_node not in hbase_allnodes:
hbase_allnodes.append(gateway_node)
HBase.stopHBaseCluster()
HBase.modifyConfig(changes=HBASE_CHANGES,
nodeSelection={'nodes': hbase_allnodes})
util.copy_back_to_original_config(HBase.getModifiedConfigPath(),
HBASE_CONF_DIR,
file_list=["hbase-site.xml"],
node_list=hbase_allnodes)
HBase.startHBaseCluster(HBase.getModifiedConfigPath())
#We grant permissions to all tables.
Phoenix.grantPermissionsToSystemTables(schemaFunctionalityEnabled=True)
#We check that we can still query the original table.
cls.verifyBasicTable()
#We check that we can create/query schemas.
exit_code, stdout = Phoenix.runSQLLineCmds(
'CREATE SCHEMA IF NOT EXISTS %s;' % SCHEMA_1)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Creation of schema %s failed due to exitcode = %s "
% (SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Schema creation %s succeeded." %
(SCHEMA_1))
#we create tables inside that schema
primaryKey = {'name': 'ID', 'type': 'BIGINT'}
columns = [{
'name': 'FirstName',
'type': 'VARCHAR(30)'
}, {
'name': 'SecondName',
'type': 'VARCHAR(30)'
}, {
'name': 'City',
'type': 'VARCHAR(30)'
}]
exit_code, stdout = Phoenix.createTable(SCHEMA_1 + '.' + TEST_TABLE_A,
primaryKey, columns)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table creation %s on schema %s failed due to exitcode = %s "
% (TEST_TABLE_A, SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table creation %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
#We insert some data into the table through upsert.
for i in range(0, 5):
exit_code, stdout = Phoenix.runSQLLineCmds(
'UPSERT INTO %s VALUES (%s, "name_%s","secondName_%s","city_%s");'
% (SCHEMA_1 + '.' + TEST_TABLE_A, str(i), str(i), str(i),
str(i)))
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table UPSERT %s on schema %s failed due to exitcode = %s "
% (TEST_TABLE_A, SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table UPSERT %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
#We verify that the data has been correctly inserted
exit_code, stdout = Phoenix.runSQLLineCmds(
'SELECT * FROM %s WHERE ID=3;' % (SCHEMA_1 + '.' + TEST_TABLE_A))
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table SELECT %s on schema %s failed due to exitcode = %s "
% (TEST_TABLE_A, SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table SELECT %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
if stdout.find('name_3') == -1 or stdout.find(
'secondName_3') == -1 or stdout.find('city_3') == -1:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table SELECT %s on schema %s returned the wrong results: %s"
% (TEST_TABLE_A, SCHEMA_1, stdout))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table SELECT %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
#We verify that we can drop the schemas with tables on it.
exit_code, stdout = Phoenix.runSQLLineCmds('DROP SCHEMA %s;' %
SCHEMA_1)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Schema drop failed due to exitcode = %s "
% (exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Schema drop succeeded.")
#We verify that the schema has been dropped.
exit_code, stdout = Phoenix.runSQLLineCmds(
'SELECT TABLE_NAME FROM SYSTEM.CATALOG WHERE SCHEMA = %s' %
SCHEMA_1,
outputFormat='xmlattr')
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Schema drop failed due to exitcode = %s "
% (exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Schema drop succeeded.")
if stdout.find(TEST_TABLE_A) != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table %s did not drop on drop schema command "
% (TEST_TABLE_A))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table %s successfuly dropped." %
TEST_TABLE_A)
# Copyright (c) 2011-2017, Hortonworks Inc. All rights reserved.
#
# Except as expressly permitted in a written agreement between your
# company and Hortonworks, Inc, any use, reproduction, modification,
# redistribution, sharing, lending or other exploitation of all or
# any part of the contents of this file is strictly prohibited.
#
#
import logging, requests, json, random, time
from beaver.config import Config
import beaver.component.dataplane.coreApi.DP_CONSTANTS as DP_CONSTANTS
from beaver.component.ambari import Ambari
logger = logging.getLogger(__name__)
DP_IP_ADDR = Config.get("dataplane", "DP_HOST_ADDR")
DP_HTTP_URL = ("https://%s/" % DP_IP_ADDR)
DP_KNOX_URL = (
"https://%s:8443/gateway/knoxsso/api/v1/websso?originalUrl=https://%s/" %
(DP_IP_ADDR, DP_IP_ADDR))
TEST_AMBARI_HOST1 = Config.get("multicluster", "AMBARI_GATEWAY1")
TEST_AMBARI_HOST2 = Config.get("multicluster", "AMBARI_GATEWAY2")
TEST_AMBARI_WEB_URL1 = Ambari.getWebUrl(hostname=TEST_AMBARI_HOST1)
TEST_AMBARI_WEB_URL2 = Ambari.getWebUrl(hostname=TEST_AMBARI_HOST2)
class DP_Api(object):
def __init__(self):
self.session = requests.session()
self.session.headers['Content-Type'] = 'application/json'
self.session.verify = False
def getTempFilepath():
return os.path.join(Config.getEnv('ARTIFACTS_DIR'), 'tmp-%d' % int(999999*random.random()))
from taskreporter.taskreporter import TaskReporter
import logging
import os
import time, datetime
from selenium import webdriver
from beaver.machine import Machine
from beaver.config import Config
from beaver.seleniumHDP import Selenium
import beaver.component.hdfsUI.commonUtils as commonUtils
from beaver.component.hdfsUI.knoxLoginPage import KnoxLoginPage
logger = logging.getLogger(__name__)
USER_NAME = Config.get('hadoop', 'HADOOPQA_USER')
PASSWORD = "******"
class BasePage(object):
KNOX_HOST = Config.get('knox', 'KNOX_HOST').split(',')[0]
KNOX_PORT = '8443'
PROTOCOL = "https://"
TOPOLOGY = "ui"
if HDFS.isFederated():
TOPOLOGY = "ui_ns1"
URL_APPEND = "/gateway/%s/yarnuiv2/" % TOPOLOGY
PROXY_URL = PROTOCOL + KNOX_HOST + ":" + KNOX_PORT + URL_APPEND
driver = None
DEFAULT_TIMEOUT = 10
def getArtifactsDir(cls):
return Config.getEnv('ARTIFACTS_DIR')
class BasePage(object):
KNOX_HOST = Config.get('knox', 'KNOX_HOST').split(',')[0]
KNOX_PORT = '8443'
PROTOCOL = "https://"
TOPOLOGY = "ui"
if HDFS.isFederated():
TOPOLOGY = "ui_ns1"
URL_APPEND = "/gateway/%s/yarnuiv2/" % TOPOLOGY
PROXY_URL = PROTOCOL + KNOX_HOST + ":" + KNOX_PORT + URL_APPEND
driver = None
DEFAULT_TIMEOUT = 10
proxy = 'true'
__timeout = DEFAULT_TIMEOUT
def __init__(self, driver=None, proxy='true'):
self.proxy = proxy
if driver:
self.driver = driver
else:
self.driver = self.instantiateWebdriver()
assert self.driver, "Could not initialize selenium webdriver"
@TaskReporter.report_test()
def getBaseUrl(self):
if self.proxy == 'true' and YARN.isKnoxProxySet():
return self.PROXY_URL
else:
return YARN.getRMUrl() + "/ui2/#/cluster-overview"
def loginToknoxSSO(self):
loginPage = KnoxLoginPage(self)
assert loginPage.isLoginPage()
# Also assert that URL contains SSO Login URL
currentURL = loginPage.getCurrentURL()
assert commonUtils.getSSOLoginURL() in currentURL
# Do Login
loginPage.doLogin(USER_NAME, PASSWORD)
def getCurrentURL(self):
return self.driver.current_url
@TaskReporter.report_test()
def checkIfCurrentURLIsProxyURL(self):
url = self.getCurrentURL()
if not YARN.isKnoxProxySet() and self.PROXY_URL not in url:
return True
if self.PROXY_URL in url:
return True
return False
def deleteSession(self):
self.quitWebdriver()
def goToBasePage(self):
self.driver.get(self.getBaseUrl())
def navigateToPage(self, url):
self.driver.get(url)
def refreshCurrentPage(self):
self.driver.get(self.driver.current_url)
def getElement(self, locatorName, returnLocatorName=False):
return locatorName if returnLocatorName else self.findElement(*self.locator_dictionary[locatorName])
def getElements(self, locatorName, returnLocatorName=False):
"""
Returns all the occurences of a matching pattern available on the web page
"""
return locatorName if returnLocatorName else self.findElements(*self.locator_dictionary[locatorName])
@TaskReporter.report_test()
def findElements(self, *loc):
try:
return self.driver.find_elements(*loc)
except Exception as e:
logger.error("Caught exception: %s" % e)
return loc
@TaskReporter.report_test()
def findElement(self, *loc):
try:
return self.driver.find_element(*loc)
except (NoSuchElementException):
logger.error("Element not found")
logger.error(loc)
return loc
@TaskReporter.report_test()
def quitWebdriver(self):
try:
Selenium.quitWebDriver(self.driver)
except Exception, e:
self.driver = None
logger.warn("Ignoring webdriver quit failure")
pass
def getHadoopQAUser(cls):
return Config.get('hadoop', 'HADOOPQA_USER')
class ruPig:
_job_user = Config.getEnv("USER")
_base_hdfs_dir = '/user/%s/ru-pig' % _job_user
_SmokeInputDir = None
_queue = 'pig'
_local_workspace = Config.getEnv("WORKSPACE")
_pig_script = os.path.join(_local_workspace, 'tests', 'rolling_upgrade',
'pig', 'script.pig')
_golden_src_file = os.path.join(_local_workspace, 'tests',
'rolling_upgrade', 'pig', 'input.txt')
_artifacts_dir = Config.getEnv('ARTIFACTS_DIR')
_hdfs_input_dir = _base_hdfs_dir + '/input'
_hdfs_input_path = _hdfs_input_dir + '/input.txt'
_hdfs_smoke_input_path = _hdfs_input_dir + '/smoke_input.txt'
_hdfs_output_dir = _base_hdfs_dir + '/output'
_hdfs_smoke_output_dir = _base_hdfs_dir + '/smoke_output'
_hdfs_success_filepath = _hdfs_output_dir + "/_SUCCESS"
_process = None
_long_running_job_timeout_secs = 600
@classmethod
def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Setup for background long running job
Upload Data to HDFS before Upgrade starts
Creates /user/hrt_qa/ru-pig dir on HDFS
Creates and Upload large data file to /user/hrt_qa/ru-pig/input/
:param runSmokeTestSetup: Runs smoke test setup if set to true
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Running Pig BackGround Job Setup ####")
HDFS.deleteDirectory(cls._base_hdfs_dir)
exit_code, stdout = HDFS.createDirectory(cls._base_hdfs_dir,
user=cls._job_user,
perm=777,
force=True)
ruAssert("Pig", exit_code == 0,
'[BGJobSetup] could not create dir on hdfs.')
HDFS.createDirectory(cls._hdfs_input_dir, force=True)
srcFile = os.path.join(cls._artifacts_dir, 'pig-ru-input.txt')
if os.path.exists(srcFile):
os.remove(srcFile)
tmpFile = os.path.join(cls._artifacts_dir, 'pig-ru-tmp-input.txt')
if os.path.exists(tmpFile):
os.remove(tmpFile)
util.copyFileToAnotherFile(cls._golden_src_file, srcFile)
util.copyFileToAnotherFile(srcFile, tmpFile)
itr = 12
if Machine.isFlubber():
itr = 16
for i in range(itr):
util.copyFileToAnotherFile(srcFile, tmpFile)
util.copyFileToAnotherFile(tmpFile, srcFile)
exit_code, stdout = HDFS.copyFromLocal(srcFile, cls._hdfs_input_path)
ruAssert("Pig", exit_code == 0, '[BGJobSetup] Data Load failed')
if runSmokeTestSetup:
cls.smoke_test_setup()
@classmethod
def smoke_test_setup(cls):
'''
Setup required to run Smoke test
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress("### Running Pig Smoke Test Setup ####")
exit_code, stdout = HDFS.copyFromLocal(cls._golden_src_file,
cls._hdfs_smoke_input_path)
ruAssert("Pig", exit_code == 0,
'[SmokeSetup] Smoke Test Data Load failed')
@classmethod
def run_background_job(cls, runSmokeTestSetup=True, config=None):
'''
Runs background long running Pig Job
:param runSmokeTestSetup: Runs smoke test setup if set to true
:param config: expected configuration location
:return: Total number of long running jobs started
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Background application for Pig started ####")
runCmd = '-p \"INPUT_PATH=%s\" -p \"OUTPUT_PATH=%s\" -p \"QUEUE=%s\" %s' % (
cls._hdfs_input_path, cls._hdfs_output_dir, cls._queue,
cls._pig_script)
#If Tez is enabled, add it in runCmd
TEZ_ENABLED = Hadoop.isTez(True, False)
if TEZ_ENABLED:
runCmd = '-x \"tez\" %s' % runCmd
cls.runCommand(cls._job_user, runCmd, runInBackground=True)
return 1
@classmethod
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run smoke test for Pig
:param smoketestnumber: Used for unique output log location
'''
return cls.run_client_smoketest(config=config)
@classmethod
def background_job_teardown(cls):
'''
Cleanup for long running Pig job
'''
logger.info("TODO")
@classmethod
def verifyLongRunningJob(cls):
'''
Validate long running background job after end of all component upgrade
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Verifying Long running job for Pig ####")
# Check if the Long Running process is not None
if cls._process is not None:
# If process poll() returns None, it means the process has not finished yet.
if cls._process.poll() is None:
UpgradePerNode.reportProgress(
"### Long running job for Pig has not Finished yet. Waiting for it to complete ####"
)
# Wait for _long_running_job_timeout_secs for Job to complete
starttime = time.time()
while (starttime - time.time() <
cls._long_running_job_timeout_secs
) and cls._process.poll() is None:
time.sleep(5)
exit_code = cls._process.poll()
if exit_code is None:
logger.info("Killing Pig Long running job process '%d'" %
cls._process.pid)
Machine.killProcess(cls._process.pid)
UpgradePerNode.reportProgress(
"### [FAILED][Pig][BGJob] Long running job for Pig Failed to finish ####"
)
elif exit_code != 0:
UpgradePerNode.reportProgress(
"### [FAILED][Pig][BGJob] Long running job for Pig Failed and Exited with '%d' ####"
% exit_code)
else:
UpgradePerNode.reportProgress(
"### Long running job for Pig Finished ####")
#Check for _SUCCESS file in HDFS Path
if HDFS.fileExists(cls._hdfs_success_filepath, cls._job_user):
UpgradePerNode.reportProgress(
"### [PASSED][Pig][BGJob] Found _SUCCESS file in HDFS for Pig Long running job ####"
)
else:
UpgradePerNode.reportProgress(
"### [FAILED][Pig][BGJob] Not Found _SUCCESS file in HDFS for Pig Long running job. ####"
)
else:
UpgradePerNode.reportProgress(
"### [FAILED][Pig][BGJob] Long Running Pig Job Failed. No Process found ####"
)
@classmethod
def upgrade_master(cls, version, config=None):
'''
Upgrades Master services:
:param version: Version to be upgraded to
:param config: Config location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Pig does not have any master, so no master to upgrade ####")
@classmethod
def upgrade_slave(cls, version, node, config=None):
'''
Upgrades slave services :
:param version: Version to be upgraded to
:param node: Slave Node
:param config: Config location
:return:
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Pig does not have any slaves, so no slaves to upgrade ####")
@classmethod
def downgrade_master(cls, version, config=None):
'''
Downgrade Master services
:param version: Version to be downgraded to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Pig does not have any master, so no master to downgrade ####")
@classmethod
def downgrade_slave(cls, version, node, config=None):
'''
Downgrade slave services
:param version: version to be downgraded to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Pig does not have any slaves, so no slaves to downgrade ####")
@classmethod
def run_client_smoketest(cls, config=None, env=None):
'''
Run Smoke test after upgrading Client
:param config: Configuration location
:param env: Set Environment variables
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress("### Running Pig Smoke test ####")
runCmd = '-p \"INPUT_PATH=%s\" -p \"OUTPUT_PATH=%s\" -p \"QUEUE=%s\" %s' % (
cls._hdfs_smoke_input_path, cls._hdfs_smoke_output_dir, cls._queue,
cls._pig_script)
#If Tez is enabled, add it in runCmd
TEZ_ENABLED = Hadoop.isTez(True, False)
if TEZ_ENABLED:
runCmd = '-x \"tez\" %s' % runCmd
cls.runCommand(cls._job_user, runCmd, runInBackground=False)
UpgradePerNode.reportProgress("### Pig Smoke test Finished ####")
@classmethod
def testAfterAllSlavesRestarted(cls):
'''
Function to test upgrade is done properly after all master and slaves are upgraded for Hdfs, yarn and Hbase
:return:
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Pig does not have any slaves, so nothing to do here ####")
@classmethod
def runCommand(cls, user, cmd, runInBackground=False):
"""
Function to run pig cmd as a certain user
:param user:
:param cmd:
:return:
"""
try:
from beaver.component.pig import Pig
except ImportError:
## Import fails when Pig is not installed on this machine. Nothing to do
return 0, ""
if runInBackground:
cls._process = Pig.runas(user, cmd, runInBackground=True)
else:
exit_code, stdout = Pig.runas(user, cmd, runInBackground=False)
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
if exit_code == 0:
UpgradePerNode.reportProgress(
"### [PASSED][Pig][Smoke] Pig Smoke test passed ####")
else:
UpgradePerNode.reportProgress(
"### [FAILED][Pig][Smoke] Pig Smoke test Failed ####")
def getHDFSUser(cls):
return Config.get('hadoop', 'HDFS_USER')
class Java2:
#set the java version
_version = '1.8.0_112'
_worksapce = Config.getEnv('WORKSPACE')
#determine tools path
_tools_path = os.path.join(_worksapce, 'tools')
#determine java home
_java_home = Config.get('machine', 'QA_CODE_JAVA_HOME')
_jdk_download_url = Config.get('machine', 'JDK8_64_URL')
_jdk_download_url_7 = Config.get('machine', 'JDK7_80_URL')
_java_cmd = os.path.join(_java_home, 'bin', 'java')
#method to setup java
@classmethod
def setupJava(cls):
# if dir exists return as its already setup
if os.path.isdir(cls._java_home):
logger.info("JDK already installed, skipping setup of JDK")
return cls._java_home
# check if tarball exists or not before downloading it
tarName = "jdk-8u112-linux-x64.tar.gz"
tarballPath = os.path.join(cls._tools_path, tarName)
if not os.path.isfile(tarballPath):
# download java
assert util.downloadUrl(cls._jdk_download_url, tarballPath)
# now install java
Machine.run('chmod 755 ' + tarballPath)
Machine.run('cd ' + cls._tools_path + '; tar zxvf ' + tarballPath)
#Machine.run('echo A | .' + tarballPath + ' -noregister 2>&1')
#Machine.tarExtractAll(filepath=tarballPath, outpath = cls._tools_path, mode='r:gz')
print cls._java_home
assert os.path.isfile(cls._java_cmd)
return cls._java_home
@classmethod
def setupJava7(cls):
# if dir exists return as its already setup
# check if tarball exists or not before downloading it
tarName = "jdk-7u80-linux-x64.tar.gz"
jdk7_folder_name = "jdk1.7.0_80"
jdk7_java_home = os.path.join(cls._tools_path, jdk7_folder_name)
if os.path.isdir(jdk7_java_home):
logger.info("JDK 7 already installed, skipping setup of JDK 7")
return jdk7_java_home
tarballPath = os.path.join(cls._tools_path, tarName)
if not os.path.isfile(tarballPath):
# download java
logger.info("JDK 7 downloading")
assert util.downloadUrl(cls._jdk_download_url_7, tarballPath)
# now install java
Machine.run('chmod 755 ' + tarballPath)
Machine.run('cd ' + cls._tools_path + '; tar zxvf ' + tarballPath)
return jdk7_java_home
@classmethod
def runJar(cls, jarName, workingDir=os.getcwd()):
(exit_code, output) = Machine.run(cls._java_cmd + " -jar " + jarName,
cwd=workingDir,
env=None,
logoutput=False)
return (exit_code, output)
def getCluster(cls):
return Config.get('hadoop', 'CLUSTER')
class Maven2:
# set the maven version
_version = '3.0.4'
# determine java home
_java_home = Java2._java_home
_worksapce = Config.getEnv('WORKSPACE')
# determine tools path
_tools_path = os.path.join(_worksapce, 'tools')
# determine maven home
_maven_home = Config.get('machine', 'MAVEN_HOME')
_maven_cmd = os.path.join(_maven_home, 'bin', 'mvn')
# what url to download maven from
_maven_download_url = Config.get('machine', 'MAVEN_URL')
#method to run maven cmd
@classmethod
def run(cls,
cmd,
cwd=None,
env=None,
mavenOpts=None,
logoutput=True,
user=None):
# make sure maven is setup before its run
cls.setupMaven()
# initialize env
if not env:
env = {}
# determine if MAVEN_OPTS need to be set
if mavenOpts:
opts = os.environ.get('MAVEN_OPTS')
if not opts:
opts = mavenOpts
else:
opts = ' '.join([opts, mavenOpts])
env['MAVEN_OPTS'] = opts
env['JAVA_HOME'] = cls._java_home
env['M2_HOME'] = cls._maven_home
# print the env so we can see what we are setting
logger.info('Env for mvn cmd')
logger.info(env)
maven_cmd = "%s %s" % (cls._maven_cmd, cmd)
exit_code, stdout = Machine.run(maven_cmd,
cwd=cwd,
env=env,
logoutput=logoutput)
return exit_code, stdout
# method to setup maven
@classmethod
def setupMaven(cls):
# if dir exists return as its already setup
if not cls._maven_home:
logger.error(
"MAVEN_HOME parameter not found. It seems like maven is not installed!!"
)
sys.exit(-1)
else:
logger.info("_maven_home : %s" % cls._maven_home)
logger.info("Maven already present at %s. Skipping install" %
cls._maven_home)
#Method to update pom.xml file entry with the annotations specified - both inlcude and exclude
@classmethod
def setProjectCategories(cls, path, groupsValue, excludedGroupsValue):
try:
fh, abs_path = mkstemp()
with open(abs_path, 'w') as new_file:
with open(path) as old_file:
for line in old_file:
new_file.write(
line.replace("${testcase.include.groups}",
str(groupsValue)).replace(
"${testcase.exclude.groups}",
str(excludedGroupsValue)))
close(fh)
remove(path)
move(abs_path, path)
except:
logger.error(
"Exception occured during setProjectCategories to update pom.xml"
)
logger.error(traceback.format_exc())
def get_java_home(cls, refresh=False):
if not cls._java_home or refresh:
zookeeper_env = os.path.join(Config.get('zookeeper', 'ZK_HOME'), "conf", "zookeeper-env.sh")
cls._java_home = util.getPropertyValueFromFile(zookeeper_env, "JAVA_HOME")
return cls._java_home
from functools import wraps
###############################################################################
logger = logging.getLogger(__name__)
###############################################################################
SETUP_TIME = 60
SHORT_TEST_TIMEOUT = SETUP_TIME + 1 * 60
MEDIUM_TEST_TIMEOUT = SETUP_TIME + 3 * 60
LONG_TEST_TIMEOUT = SETUP_TIME + 9 * 60
CONF = {}
CONF['KNOX_GUEST_USERNAME'] = "******"
CONF['KNOX_GUEST_PASSWORD'] = "******"
CONF['KNOX_PROTO'] = "https"
CONF['KNOX_HOST'] = Config.get('knox', 'KNOX_HOST').split(',')[0]
CONF['AMBARI_HOST'] = Config.get("machine", "GATEWAY")
CONF['KNOX_PORT'] = 8443
CONF['AMBARI_PORT'] = 8080
CONF['KNOX_TOPO'] = "sandbox"
CONF['KNOX_BASE_URL'] = "%s://%s:%s/gateway" % (
CONF['KNOX_PROTO'], CONF['KNOX_HOST'], CONF['KNOX_PORT'])
CONF['KNOX_TOPO_URL'] = "%s/%s" % (CONF['KNOX_BASE_URL'], CONF['KNOX_TOPO'])
CONF['KNOX_WEBHDFS_URL'] = "%s/%s/webhdfs/v1/" % (CONF['KNOX_BASE_URL'],
CONF['KNOX_TOPO'])
CONF['DIRECT_WEBHDFS_URL'] = "http://%s/webhdfs/v1/" % (
HDFS.getNamenodeHttpAddress())
CONF['RANGER_KNOX_POLICY'] = None
#CONF['SRC_DIR'] = os.path.join(Config.getEnv('WORKSPACE'), 'tests', 'knox', 'knox_2')
knox_host = CONF['KNOX_HOST']
def runas(cls, user, cmd, logoutput=True):
hadoop_cmd = Config.get('hadoop', 'HADOOP_CMD')
if Config.get('hadoop', 'HADOOP_CONF_EXCLUDE') == 'False':
hadoop_cmd += " --config " + Config.get('hadoop', 'HADOOP_CONF')
hadoop_cmd += " " + cmd
return Machine.runas(user, hadoop_cmd, logoutput=logoutput)
def isRangerInstalled():
if Config.get('xasecure', 'XA_INSTALLED') == 'yes':
return True
else:
return False
def waitForNNOutOfSafemode(cls):
exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -safemode get")
while (re.search(".*Safe mode is ON",output) != None):
exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -safemode get")
time.sleep(20)
return True
class UpgradeLogger:
UPGRADE_STATUS_LOG_FILE = os.path.join(Config.getEnv('ARTIFACTS_DIR'), 'upgrade_progress.log')
# This method returns the name of upgrade log file
@classmethod
def get_progress_local_file(cls):
return cls.UPGRADE_STATUS_LOG_FILE
# This method writes info/error messages to the log file
@classmethod
def reportProgress(cls, message, is_info_message):
message = message + "\n"
timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]
if is_info_message:
logger.info(message)
util.writeToFile(
timestamp + "|INFO|" + message.encode('utf-8'), cls.UPGRADE_STATUS_LOG_FILE, isAppend=True
)
else:
logger.error(message)
util.writeToFile(
timestamp + "|ERROR|" + message.encode('utf-8'), cls.UPGRADE_STATUS_LOG_FILE, isAppend=True
)
# This method returns last 100 lines of the error text. If number of lines < 100, all lines are returned
@classmethod
def get_error_text(cls, error_text):
error_messages = error_text.split("\n")
error_lines = "======================================\n"
max_lines_to_print = 100
total_lines = error_messages.__len__()
if total_lines > max_lines_to_print:
for error_message_index in range(total_lines - max_lines_to_print, total_lines, 1):
error_lines = error_lines + error_messages[error_message_index].decode('utf-8') + '\n'
else:
for error_message_index in range(total_lines):
error_lines = error_lines + error_messages[error_message_index].decode('utf-8') + '\n'
error_lines += "\n======================================"
return error_lines
# This method returns testname and associated failure message
@classmethod
def get_stack_trace(cls, test_name, current_dir):
apiReportDirectory = os.path.join(current_dir, 'target', 'surefire-reports', 'junitreports')
uifrmReportDirectory = os.path.join(current_dir, 'target', 'surefire-reports')
pattern = '*' + UpgradeLogger.get_testclass_name(test_name) + '*.xml'
testResultFiles = util.findMatchingFiles(
apiReportDirectory, pattern
) # First search for test result xml in api framework dir
if testResultFiles.__len__() == 0:
testResultFiles = util.findMatchingFiles(
uifrmReportDirectory, pattern
) # Now search for test result xml in uifrm dir
Machine.runas(
'root', 'chmod -R 755 ' + uifrmReportDirectory
) # Provide read permissions to everyone in uifrm junit report xml directory
else:
Machine.runas(
'root', 'chmod -R 755 ' + apiReportDirectory
) # Provide read permissions to everyone in api framework report xml directory
testresult = {}
for resultFile in testResultFiles:
testresult.update(util.parseJUnitXMLResult(resultFile))
testOutput = {}
for key, value in testresult.items():
m = re.search("([^\.]*)$", key) # Extract test name
print "key : %s " % key
print "value : %s " % value
key = m.group(0)
fail_message = value['failure']
print "Final key: %s" % key
print "Final fail_msg: %s" % fail_message
if fail_message:
testOutput[str(key)] = fail_message
print testOutput
return testOutput
@classmethod
def get_testclass_name(cls, test_name):
if "#" in test_name:
return test_name.split("#")[0]
else:
return test_name
def refreshDatanodes(cls):
exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -refreshNodes")
return output
class Java(object):
_JAVA_HOME = Config.get('machine', 'JAVA_HOME')
_JAVA_CMD = os.path.join(_JAVA_HOME, 'bin', 'java')
_JAVAC_CMD = os.path.join(_JAVA_HOME, 'bin', 'javac')
_KEYTOOL_CMD = os.path.join(_JAVA_HOME, 'bin', 'keytool')
_BEAVER_JAVA_BASE_DIR = os.path.join(Config.getEnv('WORKSPACE'), 'beaver',
'java')
_BEAVER_JAVA_COMMON_SRC_DIR = os.path.join(_BEAVER_JAVA_BASE_DIR, 'common',
'src')
_BEAVER_JAVA_TEZ_EXAMPLES_DIR = os.path.join(_BEAVER_JAVA_BASE_DIR, 'tez')
_QE_UTILS_SRC_DIR = os.path.join(_BEAVER_JAVA_COMMON_SRC_DIR, 'com',
'hortonworks', 'qe', 'utils')
def __init__(self):
pass
@classmethod
def getBeaverJavaCommonSrcDir(cls):
return cls._BEAVER_JAVA_COMMON_SRC_DIR
@classmethod
def getBeaverJavaTezExamplesDir(cls):
'''
Returns WORKSPACE/beaver/java/tez
'''
return cls._BEAVER_JAVA_TEZ_EXAMPLES_DIR
@classmethod
def getQEUtilsSrcDir(cls):
return cls._QE_UTILS_SRC_DIR
@classmethod
def runJavac(cls, sourceFiles, targetDir, classPath=None):
'''
Runs javac to compile source files with current user. Class files will be in targetDir.
Returns (exit_code, stdout)
'''
if classPath is None:
classPathStr = ""
else:
classPathStr = " -classpath " + classPath
Machine.makedirs(user=None, host=None, filepath=targetDir, passwd=None)
(exit_code,
output) = Machine.run(cls._JAVAC_CMD + classPathStr + " -d %s %s" %
(targetDir, sourceFiles))
return (exit_code, output)
@classmethod
def runJava(cls, workingDir, fqClassName, classPath=None, cmdArgs=None):
if not cmdArgs:
cmdArgs = []
if classPath is None:
classPathStr = ""
else:
classPathStr = " -classpath " + classPath
cmdArgsStr = ''
for cmdArg in cmdArgs:
cmdArgsStr += ' %s' % cmdArg
(exit_code, output) = Machine.run(cls._JAVA_CMD + classPathStr +
" %s %s" % (fqClassName, cmdArgsStr),
cwd=workingDir)
return (exit_code, output)
@classmethod
def runKeyTool(cls, command, options=None):
if not options:
options = {}
cmd = " -%s" % command
for key, value in options.items():
cmd += " %s %s" % (key, value)
(exit_code, output) = Machine.run(cls._KEYTOOL_CMD + cmd)
return (exit_code, output)
@classmethod
def runOpenSslTool(cls, command, options=None):
if not options:
options = {}
cmd = " %s" % command
for key, value in options.items():
cmd += " %s %s" % (key, value)
(exit_code, output) = Machine.run('openssl' + cmd)
return (exit_code, output)
@classmethod
def createKeyStores(cls, storedir, hosts=None, certify_key=False):
if not hosts:
hosts = ['localhost']
if certify_key:
certFile = os.path.join(storedir, 'ca-cert')
keyFile = os.path.join(storedir, 'ca-key')
exit_code, _output = cls.runOpenSslTool(
"req", {
"-nodes":
"",
"-new":
"",
"-newkey":
"rsa:2048",
"-x509":
"",
"-keyout":
keyFile,
"-out":
certFile,
"-days":
365,
"-subj":
"'/CN=myclusterxxx.hwx.site/OU=HWX/O=QE/L=Santa Clara/ST=CA/C=US'"
})
assert exit_code == 0, "Failed to generate CA"
clientTruststoreFile = os.path.join(storedir, "truststore.jks")
exit_code, _output = cls.runKeyTool(
"import", {
"-keystore": clientTruststoreFile,
"-alias": "CARoot",
"-file": certFile,
"-storepass": "******",
"-noprompt": ""
})
assert exit_code == 0, "Failed to add CA to clients trustore"
serverTruststoreFile = os.path.join(storedir,
"server.truststore.jks")
exit_code, _output = cls.runKeyTool(
"import", {
"-keystore": serverTruststoreFile,
"-alias": "CARoot",
"-file": certFile,
"-storepass": "******",
"-noprompt": ""
})
assert exit_code == 0, "Failed to add CA to servers trustore"
for host in hosts:
keystoreFile = os.path.join(storedir, "keystore_%s.jks" % host)
exit_code, _output = cls.runKeyTool(
"genkey", {
"-alias":
"example.com_%s" % host,
"-keyalg":
"RSA",
"-storetype":
"JKS",
"-keysize":
"2048",
"-storepass":
"******",
"-keypass":
"******",
"-keystore":
keystoreFile,
"-dname":
"\"CN=%s, OU=HWX, O=QE, L=Santa Clara, ST=CA, C=US\"" %
host
})
assert exit_code == 0, "Failed to generate key"
if certify_key:
certReqFile = os.path.join(storedir, '%s-cert-file' % host)
exit_code, _output = cls.runKeyTool(
"certreq", {
"-keystore": keystoreFile,
"-alias": "example.com_%s" % host,
"-file": certReqFile,
"-storepass": "******",
"-keypass": "******"
})
assert exit_code == 0, "Failed to generate certificate sign request"
certOutFile = os.path.join(storedir, '%s-cert-signed' % host)
exit_code, _output = cls.runOpenSslTool(
"x509", {
"-req": "",
"-CA": certFile,
"-CAkey": keyFile,
"-in": certReqFile,
"-out": certOutFile,
"-days": 10,
"-CAcreateserial": "",
"-passin": "pass:password"
})
exit_code, _output = cls.runKeyTool(
"import", {
"-keystore": keystoreFile,
"-alias": "CARoot",
"-file": certFile,
"-storepass": "******",
"-noprompt": ""
})
assert exit_code == 0, "Failed to import CA"
exit_code, _output = cls.runKeyTool(
"import", {
"-keystore": keystoreFile,
"-alias": "example.com_%s" % host,
"-file": certOutFile,
"-storepass": "******",
"-noprompt": ""
})
assert exit_code == 0, "Failed to import certificate"
else:
certFile = os.path.join(storedir, "%s.crt" % host)
exit_code, _output = cls.runKeyTool(
"export", {
"-alias": "example.com_%s" % host,
"-storepass": "******",
"-keystore": keystoreFile,
"-file": certFile
})
assert exit_code == 0, "Failed to export certificate"
truststoreFile = os.path.join(storedir, "truststore.jks")
exit_code, _output = cls.runKeyTool(
"import", {
"-trustcacerts": "",
"-alias": "example.com_%s" % host,
"-noprompt": "",
"-storepass": "******",
"-keystore": truststoreFile,
"-file": certFile
})
assert exit_code == 0, "Failed to import certificate"
def runInBackgroundAs(cls, user, cmd):
hadoop_cmd = Config.get('hadoop', 'HADOOP_CMD')
if Config.get('hadoop', 'HADOOP_CONF_EXCLUDE') == 'False':
hadoop_cmd += " --config " + Config.get('hadoop', 'HADOOP_CONF')
hadoop_cmd += " " + cmd
return Machine.runinbackgroundAs(user, hadoop_cmd)
def updateProperties():
if CommonLib.is_ambari_security_enabled(Config.get('ambari', 'AMBARI_SERVER_HTTPS')
) or CommonLib.is_ambari_security_enabled(Config.get('ambari',
'WIRE_ENCRYPTION')):
port = 8443
https = "yes"
else:
port = 8080
https = "no"
DEFAULT_TEST_TIMEOUT = 45 # default value of 45 minutes, will extend this later to allow as an input through ADDITIONAL_AMBARI_PROPS if user wants to
#populate the config properties file for api test code
configPropChanges = {}
configPropChanges['HOST'] = Config.get('ambari', 'HOST')
configPropChanges['PORT'] = port
# Below check is only done for API framework as part of post Upgrade, hence a safe assumption to use STACKNAME as HDP (since the calls are made after upgrade from BigInsights to HDP)
if 'BigInsights'.lower() in Config.get('ambari', 'STACK_TYPE').lower():
logger.info("STACK_TYPE = BigInsights, changing to HDP for post upgrade operations")
configPropChanges['STACKNAME'] = 'HDP'
else:
configPropChanges['STACKNAME'] = Config.get('ambari', 'STACK_TYPE')
configPropChanges['STACKVERSION_TO_UPGRADE'] = Config.get('ambari', 'STACK_UPGRADE_TO')
configPropChanges['BUILDNUMBER_TO_UPGRADE'] = Config.get('ambari', 'STACK_UPGRADE_TO')
configPropChanges['MOTD_ENABLE'] = Config.get('ambari', 'MOTD_ENABLE')
configPropChanges['AMBARI_DB'] = Config.get('ambari', 'AMBARI_DB')
configPropChanges['AMBARI_SERVER_HTTPS'] = https
configPropChanges['SECURITY'] = Config.get('machine', 'IS_SECURE')
configPropChanges['KERBEROS_SERVER_TYPE'] = Config.get('ambari', 'KERBEROS_SERVER_TYPE')
configPropChanges['REALM'] = Config.get('ambari', 'REALM')
configPropChanges['USER_KERBEROS_SERVER_TYPE'] = Config.get('ambari', 'USER_KERBEROS_SERVER_TYPE')
configPropChanges['AD_SERVER_HOST'] = Config.get('ambari', 'AD_SERVER_HOST')
configPropChanges['USER_REALM'] = Config.get('ambari', 'USER_REALM')
configPropChanges['CUSTOMIZED_SERVICES_USERS'] = Config.get('ambari', 'CUSTOMIZED_SERVICES_USERS')
configPropChanges['AMBARI_2WAY_SSL'] = Config.get('ambari', 'AMBARI_2WAY_SSL')
configPropChanges['WIRE_ENCRYPTION'] = Config.get('ambari', 'WIRE_ENCRYPTION')
configPropChanges['DEFAULT_TESTCASE_TIMEOUT'] = DEFAULT_TEST_TIMEOUT
configPropChanges['RUN_INSTALLER'] = Config.get('machine', 'RUN_INSTALLER')
# Added to read additional ambari props esp. for Patch Upgrade related properties like SERVICES_TO_UPGRADE, UPGRADE_TYPE
logger.info("Checking if any ADDITIONAL_AMBARI_PROPS need to be updated")
ADDITIONAL_AMBARI_PROPS = Config.get('ambari', 'ADDITIONAL_AMBARI_PROPS')
if ADDITIONAL_AMBARI_PROPS:
parameter_map = ADDITIONAL_AMBARI_PROPS.split(",")
for parameter in parameter_map:
key_value = parameter.split("=")
key = key_value[0]
value = key_value[1]
print "Reading key :%s = value :%s" % (key, value)
configPropChanges[key] = value
return configPropChanges
def refreshTasktrackers(cls):
exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "mradmin -refreshNodes")
return output
def read_job_properties(cls, local_work_dir):
propChanges = {}
COMPONENT = Config.get('ambari', 'COMPONENT')
if 'deploytest' in COMPONENT:
propChanges['INSTALL_HDP'] = "true"
propChanges['MR_FRAMEWORK'] = Config.get('ambari', 'MR_FRAMEWORK')
propChanges['HOST'] = Config.get('ambari', 'HOST') + ":8080"
propChanges['CLUSTER_NAME'] = Config.get('ambari', 'CLUSTER_NAME')
propChanges['STACK'] = Config.get('ambari', 'STACK')
propChanges['UMASK'] = Config.get('ambari', 'UMASK')
propChanges['AMBARI_CUSTOM_USER'] = Config.get('ambari', 'AMBARI_CUSTOM_USER')
propChanges['AMBARI_AGENT_USER'] = Config.get('ambari', 'AMBARI_AGENT_USER')
propChanges['PWD_ENCRYPT'] = Config.get('ambari', 'PWD_ENCRYPT')
propChanges['CUSTOMIZED_SERVICES_USERS'] = Config.get('ambari', 'CUSTOMIZED_SERVICES_USERS')
propChanges['CUSTOM_PIDS'] = Config.get('ambari', 'CUSTOM_PIDS')
propChanges['AMBARI_2WAY_SSL'] = Config.get('ambari', 'AMBARI_2WAY_SSL')
propChanges['AMBARI_SERVER_HTTPS'] = Config.get('ambari', 'AMBARI_SERVER_HTTPS')
propChanges['IS_TMP_NOEXEC'] = Config.get('ambari', 'IS_TMP_NOEXEC')
propChanges['NUMBER_OF_BASE_HOSTS'] = Config.get('ambari', 'NUMBER_OF_BASE_HOSTS')
propChanges['DN_NONROOT'] = Config.get('ambari', 'DN_NONROOT')
propChanges['MOTD_ENABLE'] = Config.get('ambari', 'MOTD_ENABLE')
propChanges['HDP_REPO_BASEURL'] = Config.get('ambari', 'HDP_REPO_BASEURL')
propChanges['HDP_UTILS_REPO_BASEURL'] = Config.get('ambari', 'HDP_UTILS_REPO_BASEURL')
propChanges['AMBARI_DB'] = Config.get('ambari', 'AMBARI_DB')
propChanges['JDK_VERSION'] = Config.get('ambari', 'JDK_VERSION')
propChanges['IS_SECURE'] = Config.get('machine', 'IS_SECURE')
propChanges['CLIENT'] = Config.get('ambari', 'CLIENT')
propChanges['CLIENT_OS'] = Config.get('ambari', 'CLIENT_OS')
propChanges['CLIENT_PORT'] = Config.get('ambari', 'CLIENT_PORT')
propChanges['UPGRADE_TO'] = Config.get('ambari', 'UPGRADE_TO')
propChanges['STACK_UPGRADE_TO'] = Config.get('ambari', 'STACK_UPGRADE_TO')
propChanges['VIDEO_RECORDING'] = Config.get('ambari', 'VIDEO_RECORDING')
propChanges['BROWSER'] = Config.get('ambari', 'BROWSER')
propChanges['AMBARI_RPM_URL'] = Config.get('ambari', 'AMBARI_VERSION')
propChanges['HIVE_DB'] = Config.get('ambari', 'DATABASE_FLAVOR')
propChanges['DRUID_DB'] = Config.get('ambari', 'DATABASE_FLAVOR')
propChanges['BEACON_DB'] = Config.get('ambari', 'DATABASE_FLAVOR')
propChanges['OOZIE_DB'] = Config.get('ambari', 'DATABASE_FLAVOR')
propChanges['XA_DATABASE_FLAVOR'] = Config.get('ambari', 'XA_DATABASE_FLAVOR')
propChanges['IS_HA_TEST'] = Config.get('ambari', 'IS_HA_TEST')
propChanges['ENABLE_HA_COMPONENTS'] = Config.get('ambari', 'ENABLE_HA_COMPONENTS')
propChanges['USE_BLUEPRINT'] = Config.get('ambari', 'USE_BLUEPRINT')
propChanges['USER_KERBEROS_SERVER_TYPE'] = Config.get('ambari', 'USER_KERBEROS_SERVER_TYPE')
propChanges['KERBEROS_SERVER_TYPE'] = Config.get('ambari', 'KERBEROS_SERVER_TYPE')
propChanges['REALM'] = Config.get('ambari', 'REALM')
propChanges['USER_REALM'] = Config.get('ambari', 'USER_REALM')
propChanges['AD_SERVER_HOST'] = Config.get('ambari', 'AD_SERVER_HOST')
propChanges['WIRE_ENCRYPTION'] = Config.get('ambari', 'WIRE_ENCRYPTION')
propChanges['SPLIT_NUM'] = Config.get('ambari', 'SPLIT_NUM')
propChanges['AMBARI_TESTSNAMES'] = Config.get('ambari', 'AMBARI_TESTSNAMES')
propChanges['RUN_MARKER_LIST'] = Config.get('ambari', 'RUN_MARKER_LIST')
propChanges['RUN_MARKER_VERSION'] = Config.get('ambari', 'RUN_MARKER_VERSION')
propChanges['STACK_TYPE'] = Config.get('ambari', 'STACK_TYPE')
propChanges['HDF_REPO_BASEURL'] = Config.get('ambari', 'HDF_REPO_BASEURL')
propChanges['MANAGEMENT_PACK_LINK'] = Config.get('ambari', 'MANAGEMENT_PACK_LINK')
propChanges['STREAMLINE_DB'] = Config.get('ambari', 'STREAMLINE_DB')
propChanges['REGISTRY_DB'] = Config.get('ambari', 'REGISTRY_DB')
propChanges['USE_BAKED_IMAGE'] = Config.get('ambari', 'USE_BAKED_IMAGE') # QE-18454
propChanges['INSTALL_MR'] = cls.convert_environment_variable_to_bool("INSTALL_MR")
propChanges['INSTALL_ZOOKEEPER'] = cls.convert_environment_variable_to_bool("INSTALL_ZOOKEEPER")
propChanges['INSTALL_AMS'] = cls.convert_environment_variable_to_bool("INSTALL_AMS")
propChanges['INSTALL_KMS'] = cls.convert_environment_variable_to_bool("INSTALL_KMS")
propChanges['INSTALL_NFSGATEWAYS'] = cls.convert_environment_variable_to_bool("INSTALL_NFSGATEWAYS")
propChanges['INSTALL_HS_INTERACTIVE'] = cls.convert_environment_variable_to_bool("INSTALL_HS_INTERACTIVE")
propChanges['INSTALL_LOGSEARCH'] = cls.convert_environment_variable_to_bool("INSTALL_LOGSEARCH")
propChanges['INSTALL_FALCON'] = cls.convert_environment_variable_to_bool("INSTALL_FALCON")
propChanges['INSTALL_STREAMLINE'] = cls.convert_environment_variable_to_bool("INSTALL_STREAMLINE")
propChanges['INSTALL_SPARK'] = cls.convert_environment_variable_to_bool("INSTALL_SPARK")
propChanges['INSTALL_PDSH'] = cls.convert_environment_variable_to_bool("INSTALL_PDSH")
propChanges['INSTALL_PIG'] = cls.convert_environment_variable_to_bool("INSTALL_PIG")
propChanges['INSTALL_HBASE'] = cls.convert_environment_variable_to_bool("INSTALL_HBASE")
propChanges['INSTALL_TEMPLETON'] = cls.convert_environment_variable_to_bool("INSTALL_TEMPLETON")
propChanges['INSTALL_OOZIE'] = cls.convert_environment_variable_to_bool("INSTALL_OOZIE")
propChanges['INSTALL_HIVE'] = cls.convert_environment_variable_to_bool("INSTALL_HIVE")
propChanges['INSTALL_HCAT'] = cls.convert_environment_variable_to_bool("INSTALL_HCAT")
propChanges['INSTALL_SQOOP'] = cls.convert_environment_variable_to_bool("INSTALL_SQOOP")
propChanges['INSTALL_SQOOP2'] = cls.convert_environment_variable_to_bool("INSTALL_SQOOP2")
propChanges['INSTALL_FLUME'] = cls.convert_environment_variable_to_bool("INSTALL_FLUME")
propChanges['INSTALL_STORM'] = cls.convert_environment_variable_to_bool("INSTALL_STORM")
propChanges['INSTALL_KNOX'] = cls.convert_environment_variable_to_bool("INSTALL_KNOX")
propChanges['INSTALL_KAFKA'] = cls.convert_environment_variable_to_bool("INSTALL_KAFKA")
propChanges['INSTALL_HBASE_APP'] = cls.convert_environment_variable_to_bool("INSTALL_HBASE_APP")
propChanges['INSTALL_STORM_APP'] = cls.convert_environment_variable_to_bool("INSTALL_STORM_APP")
propChanges['INSTALL_ACCUMULO_APP'] = cls.convert_environment_variable_to_bool("NSTALL_ACCUMULO_APP")
propChanges['INSTALL_PHOENIX_APP'] = cls.convert_environment_variable_to_bool("INSTALL_PHOENIX_APP")
propChanges['INSTALL_MAHOUT'] = cls.convert_environment_variable_to_bool("INSTALL_MAHOUT")
propChanges['INSTALL_SMARTSENSE'] = cls.convert_environment_variable_to_bool("INSTALL_SMARTSENSE")
propChanges['INSTALL_PHOENIX'] = cls.convert_environment_variable_to_bool("INSTALL_PHOENIX")
propChanges['INSTALL_RANGER'] = cls.convert_environment_variable_to_bool("INSTALL_XASECURE")
propChanges['INSTALL_SLIDER'] = cls.convert_environment_variable_to_bool("INSTALL_SLIDER")
propChanges['INSTALL_ACCUMULO'] = cls.convert_environment_variable_to_bool("INSTALL_ACCUMULO")
propChanges['INSTALL_HUE'] = cls.convert_environment_variable_to_bool("INSTALL_HUE")
propChanges['INSTALL_ATLAS'] = cls.convert_environment_variable_to_bool("INSTALL_ATLAS")
propChanges['INSTALL_HDF'] = cls.convert_environment_variable_to_bool("INSTALL_HDF")
propChanges['INSTALL_NIFI'] = cls.convert_environment_variable_to_bool("INSTALL_NIFI")
propChanges['INSTALL_AMBARI_INFRA'] = cls.convert_environment_variable_to_bool("INSTALL_AMBARI_INFRA")
propChanges['INSTALL_JNBG'] = cls.convert_environment_variable_to_bool("INSTALL_JNBG")
propChanges['INSTALL_R4ML'] = cls.convert_environment_variable_to_bool("INSTALL_R4ML")
propChanges['INSTALL_TITAN'] = cls.convert_environment_variable_to_bool("INSTALL_TITAN")
propChanges['INSTALL_SOLR'] = cls.convert_environment_variable_to_bool("INSTALL_SOLR")
propChanges['INSTALL_REGISTRY'] = cls.convert_environment_variable_to_bool("INSTALL_REGISTRY")
propChanges['INSTALL_ZEPPELIN'] = cls.convert_environment_variable_to_bool("INSTALL_ZEPPELIN")
propChanges['INSTALL_SPARK2'] = cls.convert_environment_variable_to_bool("INSTALL_SPARK2")
propChanges['INSTALL_DRUID'] = cls.convert_environment_variable_to_bool("INSTALL_DRUID")
propChanges['INSTALL_BEACON'] = cls.convert_environment_variable_to_bool("INSTALL_BEACON")
propChanges['MR_FRAMEWORK'] = cls.convert_environment_variable_to_bool("MR_FRAMEWORK")
propChanges['INSTALL_LZO'] = cls.convert_environment_variable_to_bool("INSTALL_LZO")
propChanges['ENABLE_KNOX_SSO'] = cls.convert_environment_variable_to_bool("ENABLE_KNOX_SSO") # QE-19395
logger.info("the prop changes" + str(propChanges))
ADDITIONAL_AMBARI_PROPS = Config.get('ambari', 'ADDITIONAL_AMBARI_PROPS')
if ADDITIONAL_AMBARI_PROPS:
parameter_map = ADDITIONAL_AMBARI_PROPS.split(",")
for parameter in parameter_map:
key_value = parameter.split("=")
key = key_value[0]
value = key_value[1]
print "Reading key :%s = value :%s" % (key, value)
propChanges[key] = value
if 'sanity' in COMPONENT and not 'sanity-preupgrade' in COMPONENT or 'postupg-sec-enable' in COMPONENT:
stack_upgrade_to = Config.get('ambari', 'STACK_UPGRADE_TO')
if stack_upgrade_to is not None and len(stack_upgrade_to) > 0:
propChanges['STACK'] = Config.get('ambari', 'STACK_UPGRADE_TO')
if not propChanges['UPGRADE_TO'] is None:
logger.info("Check value " + propChanges['UPGRADE_TO'])
# If the gateway is not deployed, change /root/hosts to have only those hosts that are to be part of gateway cluster
if CommonLib.update_hosts_for_deployed_clusters(propChanges):
propChanges['INSTALL_HDP'] = "true"
util.writePropertiesToFile(
os.path.join(local_work_dir, 'ambari.properties'), os.path.join(local_work_dir, 'ambari.properties'),
propChanges
)
log_prop_changes = {}
log_prop_changes['log4j.appender.UIFRM.File'] = os.path.join(local_work_dir, "uifrm.log")
util.writePropertiesToFile(
os.path.join(local_work_dir, 'log4j.properties'), os.path.join(local_work_dir, 'log4j.properties'),
log_prop_changes
)
return propChanges
def getTasktrackers(cls):
slaveFile = os.path.join(Config.get('hadoop', 'HADOOP_CONF'),"slaves")
f = open(slaveFile, "r")
tasktrackers = f.readlines()
return tasktrackers
def get_component(cls):
return Config.get('ambari', 'COMPONENT')
def common_startup_operations(cls, local_work_dir):
from tools.artifact_server_helper import artifact_server_start
artifact_server_start(Config.getEnv('ARTIFACTS_DIR'))
Machine.runas('root', 'chmod -R 755 ' + os.path.join(local_work_dir, 'target'))
#Get user from config file
HADOOPQA_USER = CommonHadoopEnv.getHadoopQAUser()
HDFS_USER = CommonHadoopEnv.getHDFSUser()
MAPRED_USER = CommonHadoopEnv.getMapredUser()
SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
DATA_PATH = os.path.join(SCRIPT_PATH, "data")
CREATE_FILE = "CreateFile.py"
CREATE_FILE_PATH_IN_HADOOP = '/user/' + HADOOPQA_USER + '/' + CREATE_FILE
CREATE_FILE_PATH_IN_LOCAL = os.path.join(SCRIPT_PATH, "data", CREATE_FILE)
CREATE_FILE_2 = "CreateFile2.py"
CREATE_FILE_2_PATH_IN_HADOOP = '/user/' + HADOOPQA_USER + '/' + CREATE_FILE_2
CREATE_FILE_2_PATH_IN_LOCAL = os.path.join(SCRIPT_PATH, "data", CREATE_FILE_2)
OUT_PATH_IN_HADOOP = '/user/' + HADOOPQA_USER + '/out1'
HADOOP_STREAMING_JAR = Config.get('hadoop', 'HADOOP_STREAMING_JAR')
logger = logging.getLogger(__name__)
def validateJobId(jobId):
jobId = jobId + ""
return jobId.startswith('job_')
def getLocalDirInfo(host):
return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "mapred-site.xml"), "mapred.local.dir")
def checkJobCreatedTempFileInTT(logFileDir, currentUser, currentJobId, currentAttemptId, logfile, taskTrackerHost):
pathFile = os.path.join(logFileDir, 'taskTracker', currentUser, 'jobcache', currentJobId, currentAttemptId, 'work', logfile)
logger.info("path file: " + pathFile)
result = False
if platform.system() == 'Windows':
