def updateJobProperties(cls,
propFile,
properties=None,
haEnabled=False,
debug=False):
fileSystemName = Hadoop.getFSDefaultValue()
jobTrackerIP = MAPRED.getJobtrackerAddress()
jobTracker = jobTrackerIP[0] + ":" + jobTrackerIP[1]
if not properties:
properties = {}
if not properties.has_key('nameNode'):
properties['nameNode'] = fileSystemName
if not properties.has_key('jobTracker'):
properties['jobTracker'] = jobTracker
if "hcatalog" in propFile:
if Hadoop.isSecure():
kerberosPrincipal = Hive.getConfigValue(
"hive.metastore.kerberos.principal")
properties[
'hive.metastore.kerberos.principal'] = kerberosPrincipal
logger.info("Updating for hcatalog workflow")
hcatNode = Hive.getConfigValue("hive.metastore.uris").replace(
'thrift', 'hcat')
logger.info("Hcat node is " + hcatNode)
properties['hcatNode'] = hcatNode
if Hadoop.isSecure():
# determine the namenode and the jobtracker principal
nnPrincipal = None
if haEnabled:
nnPrincipal = HDFS.getNameNodePrincipal().replace(
'_HOST', HDFS.getNamenodeByState('active'))
else:
nnPrincipal = HDFS.getNameNodePrincipal().replace(
'_HOST',
HDFS.getNamenodeHttpAddress()[0])
jtPrincipal = MAPRED.getMasterPrincipal().replace(
'_HOST', jobTrackerIP[0])
properties['dfs.namenode.kerberos.principal'] = nnPrincipal
properties['mapreduce.jobtracker.kerberos.principal'] = jtPrincipal
wfPath = util.getPropertyValueFromFile(propFile,
"oozie.wf.application.path")
if wfPath != None and wfPath.find("hdfs://localhost:9000") != -1:
wfPath = wfPath.replace("hdfs://localhost:9000", fileSystemName)
logger.info("Value of replaced oozie.wf.application.path is " +
wfPath)
properties['oozie.wf.application.path'] = wfPath
util.writePropertiesToFile(propFile, propFile, properties)
if debug:
logger.info('Content of properties file %s' % propFile)
f = open(propFile, 'r')
# print the file to the console
logger.info(f.read())
f.close()
def share_lib_setup(cls, fs=None):
# only need to do this on windows
if not Machine.isWindows():
# return exit code of 0 and empty stdout
return 0, ''
oozie_setup_cmd = os.path.join(OOZIE_HOME, 'bin', 'oozie-setup.cmd')
oozie_sharelib_location = os.path.join(OOZIE_HOME, "share")
if not fs:
fs = Hadoop.getFSDefaultValue()
cmd = 'sharelib create -fs %s -locallib %s' % (fs,
oozie_sharelib_location)
return cls.runas(cls._oozie_user, cmd, base_cmd=oozie_setup_cmd)
def createClusterEntities(cls, colo, desc, name):
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
from beaver.component.hadoop import Hadoop, HDFS, YARN
write_endpoint = Hadoop.getFSDefaultValue()
webhdfs_scheme = 'webhdfs'
if HDFS.isHttpsEnabled():
webhdfs_scheme = 'swebhdfs'
read_endpoint = '%s://%s:%s' % (
webhdfs_scheme, write_endpoint.split('/')[2].split(':')[0],
HDFS.getNNWebPort())
execute_endpoint = YARN.getResourceManager()
falconNode = Falcon.get_falcon_server()
from beaver.component.oozie import Oozie
oozieUrl = Oozie.getOozieUrl()
entityText = "<?xml version=\"1.0\"?>" \
"<cluster colo=\"" + colo + "\" description=\"" + desc + "\" name=\"" + name + "\" " \
"xmlns=\"uri:falcon:cluster:0.1\"> " \
"<interfaces> " \
"<interface type=\"readonly\" endpoint=\""+read_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"write\" endpoint=\""+write_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"execute\" endpoint=\"" + execute_endpoint + "\" version=\"0.20.2\"/> " \
"<interface type=\"workflow\" endpoint=\"" + oozieUrl + "\" version=\"3.1\"/>" \
"<interface type=\"messaging\" endpoint=\"" \
"tcp://" + falconNode + ":61616?daemon=true\" version=\"5.1.6\"/>" \
"</interfaces>" \
"<locations>" \
"<location name=\"staging\" path=\"/apps/falcon/" + name + "/staging\" />" \
"<location name=\"temp\" path=\"/tmp\" />" \
"<location name=\"working\" path=\"/apps/falcon/" + name + "/working\" />" \
"</locations>" \
"<ACL owner=\"" + cls._job_user + "\" group=\"users\" permission=\"0755\"/>"
if Hadoop.isSecure():
realm = HDFS.getConfigValue(
'dfs.namenode.kerberos.principal').split('@')[1]
entityText += "<properties> <property name=\"dfs.namenode.kerberos.principal\" value=\"nn/_HOST@" + realm + "\"/> </properties>"
entityText += "</cluster>"
textFile = open(os.path.join(cls._local_workspace, name + ".xml"), "w")
textFile.write("%s" % entityText)
textFile.close()
return
def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Setup for background long running job
:param runSmokeTestSetup: Runs smoke test setup if set to true
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][FLUME][BGJobSetup] Long running job setup for Flume component started"
)
from beaver.component.flume import Agent
global agent1
global agent2
if not os.path.exists(cls._local_work_dir):
os.mkdir(cls._local_work_dir)
shutil.copy(cls._flume_datagen_src, cls._local_work_dir)
agent1 = Agent(cls._local_work_dir)
agent2 = Agent(cls._local_work_dir)
for outdir in (cls._agent1_chkpt_dir, cls._agent1_data_dir,
cls._agent2_chkpt_dir, cls._agent2_data_dir):
os.mkdir(outdir)
logger.info("Preparing the Flume configs for long running test")
propertyMap = {}
namenode = Hadoop.getFSDefaultValue()
propertyMap['agent2.sinks.hdfsSink.hdfs.path'] = "%s%s" % (
namenode, cls._hdfs_test_dir)
if Hadoop.isSecure():
if Config.hasOption('machine', 'USER_REALM'):
user_realm = Config.get('machine', 'USER_REALM', '')
else:
nnKerbPrincipal = HDFS.getNameNodePrincipal(defaultValue='')
atloc = nnKerbPrincipal.find("@")
if atloc != -1:
user_realm = nnKerbPrincipal[atloc:]
if user_realm:
propertyMap[
'agent2.sinks.hdfsSink.hdfs.kerberosPrincipal'] = cls._test_user + '@' + user_realm
propertyMap[
'agent2.sinks.hdfsSink.hdfs.kerberosKeytab'] = Machine.getHeadlessUserKeytab(
cls._test_user)
util.writePropertiesToFile(
os.path.join(cls._flume_test_conf, 'longrunning.properties'),
cls._flume_test_src, propertyMap)
def getEnvironmentVariables(cls, directory):
# Need to set the below for the oozie-regression test
OOZIE_HDFS_LOG_DIR = str(
Hadoop.getFSDefaultValue(True) + '/user/' +
Config.get("oozie", "OOZIE_USER") + '/oozie-logs/')
if Machine.type() == "Windows":
sep = ";"
OOZIE_HDFS_LOG_DIR = OOZIE_HDFS_LOG_DIR + os.environ.get(
"COMPUTERNAME")
else:
sep = ":"
OOZIE_HDFS_LOG_DIR = OOZIE_HDFS_LOG_DIR + Oozie.getOozieServers(
)[0]
OOZIE_QA_REG_DIR = cls.getOozieRegressionFolder()
path = os.path.join(
OOZIE_QA_REG_DIR, "lib",
"yoozietest-qa-1.0.0-SNAPSHOT.jar") + sep + os.path.join(
OOZIE_QA_REG_DIR, "lib",
"original-yoozietest-qa-1.0.0-SNAPSHOT.jar")
Config.setEnv("LOCAL_CP", path)
oozie_server = cls.getOozieUrl()
if HDFS.isASV() or HDFS.isCabo():
if HDFS.isCabo():
# Cabo gets FQDN uri to use the scheme as a differentiator for other FS
HIT_NN = str(Hadoop.getConfigValue("fs.defaultFS", None))
else: # is ASV
HIT_NN = str(Hadoop.getFSDefaultValue(False))
return {
"JAVA_HOME": Config.get('machine', 'JAVA_HOME'),
"HADOOP_HOME": Config.get('hadoop', 'HADOOP_HOME'),
"HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'),
"HIT_NN": HIT_NN,
"HIT_JT": str(MAPRED.getJobtrackerAddress()[0]),
"HIT_JT_PORT": str(MAPRED.getJobtrackerAddress()[1]),
###TODO Change following 3 for secure setups
"OOZIE_QA_AUTH": "simple",
"OOZIE_NN_KRB_PRINCIPAL": "Blah",
"OOZIE_JT_KRB_PRINCIPAL": "Blah",
"OOZIE_TEST_SUITE": "testngRegressionSuiteDebug",
"OOZIE_HOME": Config.get("oozie", "OOZIE_HOME"),
"OOZIE_PORT": Config.get("oozie", "OOZIE_PORT"),
"OOZIE_SECURE_HOSTNAME": Machine.getfqdn(),
"OOZIE_FOLDER": Config.get("oozie", "OOZIE_USER"),
"OOZIE_USER": Config.get("oozie", "OOZIE_USER"),
"FIREFOX_PATH": Config.get("firefox", "FIREFOX_PATH"),
"FIREFOX_DISPLAY": Config.get("firefox", "FIREFOX_DISPLAY"),
"OOZIE_QA_REG_DIR": OOZIE_QA_REG_DIR,
"OOZIE_QA_HADOOP_QUEUE": "default",
"OOZIE_URL": str(oozie_server),
"HIT_OOZIE": ((str(oozie_server)).split(":")[2]).split("/")[0],
"LOCAL_CP": path,
"HIT_HDFS_STORAGE_DIR": directory,
"OOZIE_HDFS_LOG_DIR": OOZIE_HDFS_LOG_DIR
}
else:
if Hadoop.isSecure():
oozie_qa_auth = "kerberos"
nnPrincipal = HDFS.getNameNodePrincipal()
jtPrincipal = MAPRED.getMasterPrincipal()
user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
else:
oozie_qa_auth = "simple"
nnPrincipal = "blah"
jtPrincipal = "blah"
kerbTicket = "blah"
return {
"JAVA_HOME": Config.get('machine', 'JAVA_HOME'),
"HADOOP_HOME": Config.get('hadoop', 'HADOOP_HOME'),
"HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'),
"HIT_NN": str(Hadoop.getFSDefaultValue(False)),
"HIT_NN_PORT":
str(Hadoop.getFSDefaultValue(True)).split(":")[2],
"HIT_JT": str(MAPRED.getJobtrackerAddress()[0]),
"HIT_JT_PORT": str(MAPRED.getJobtrackerAddress()[1]),
"OOZIE_QA_AUTH": oozie_qa_auth,
"OOZIE_NN_KRB_PRINCIPAL": nnPrincipal,
"OOZIE_JT_KRB_PRINCIPAL": jtPrincipal,
"OOZIE_TEST_SUITE": "testngRegressionSuiteDebug",
"OOZIE_HOME": Config.get("oozie", "OOZIE_HOME"),
"OOZIE_PORT": Config.get("oozie", "OOZIE_PORT"),
"OOZIE_FOLDER": Config.get("oozie", "OOZIE_USER"),
"OOZIE_USER": Config.get("oozie", "OOZIE_USER"),
"FIREFOX_PATH": Config.get("firefox", "FIREFOX_PATH"),
"FIREFOX_DISPLAY": Config.get("firefox", "FIREFOX_DISPLAY"),
"OOZIE_QA_REG_DIR": OOZIE_QA_REG_DIR,
"OOZIE_QA_HADOOP_QUEUE": "default",
"OOZIE_URL": str(oozie_server),
"HIT_OOZIE": ((str(oozie_server)).split(":")[2]).split("/")[0],
"LOCAL_CP": path,
"HIT_HDFS_STORAGE_DIR": directory,
"KRB5CCNAME": kerbTicket,
"OOZIE_HDFS_LOG_DIR": OOZIE_HDFS_LOG_DIR
}
