def updateJobProperties(cls,
propFile,
properties=None,
haEnabled=False,
debug=False):
fileSystemName = Hadoop.getFSDefaultValue()
jobTrackerIP = MAPRED.getJobtrackerAddress()
jobTracker = jobTrackerIP[0] + ":" + jobTrackerIP[1]
if not properties:
properties = {}
if not properties.has_key('nameNode'):
properties['nameNode'] = fileSystemName
if not properties.has_key('jobTracker'):
properties['jobTracker'] = jobTracker
if "hcatalog" in propFile:
if Hadoop.isSecure():
kerberosPrincipal = Hive.getConfigValue(
"hive.metastore.kerberos.principal")
properties[
'hive.metastore.kerberos.principal'] = kerberosPrincipal
logger.info("Updating for hcatalog workflow")
hcatNode = Hive.getConfigValue("hive.metastore.uris").replace(
'thrift', 'hcat')
logger.info("Hcat node is " + hcatNode)
properties['hcatNode'] = hcatNode
if Hadoop.isSecure():
# determine the namenode and the jobtracker principal
nnPrincipal = None
if haEnabled:
nnPrincipal = HDFS.getNameNodePrincipal().replace(
'_HOST', HDFS.getNamenodeByState('active'))
else:
nnPrincipal = HDFS.getNameNodePrincipal().replace(
'_HOST',
HDFS.getNamenodeHttpAddress()[0])
jtPrincipal = MAPRED.getMasterPrincipal().replace(
'_HOST', jobTrackerIP[0])
properties['dfs.namenode.kerberos.principal'] = nnPrincipal
properties['mapreduce.jobtracker.kerberos.principal'] = jtPrincipal
wfPath = util.getPropertyValueFromFile(propFile,
"oozie.wf.application.path")
if wfPath != None and wfPath.find("hdfs://localhost:9000") != -1:
wfPath = wfPath.replace("hdfs://localhost:9000", fileSystemName)
logger.info("Value of replaced oozie.wf.application.path is " +
wfPath)
properties['oozie.wf.application.path'] = wfPath
util.writePropertiesToFile(propFile, propFile, properties)
if debug:
logger.info('Content of properties file %s' % propFile)
f = open(propFile, 'r')
# print the file to the console
logger.info(f.read())
f.close()
def share_lib_update(cls, fs=None):
# only need to do this on windows
oozie_server = cls.getOozieUrl()
cmd = " admin -oozie " + oozie_server + " -sharelibupdate"
if Hadoop.isSecure():
cmd = " -Doozie.auth.token.cache=false " + cmd
return cls.runas(cls._oozie_user, cmd)
def submit_storm_hive_topology(cls, tcId, className, args,
useStandaloneCmd):
if Hadoop.isSecure():
if Config.hasOption('machine', 'USER_REALM'):
user_realm = Config.get('machine', 'USER_REALM', '')
else:
nnKerbPrincipal = HDFS.getNameNodePrincipal(defaultValue='')
atloc = nnKerbPrincipal.find("@")
if (atloc != -1):
user_realm = nnKerbPrincipal[atloc:]
if user_realm != None:
args += " " + Machine.getHeadlessUserKeytab(
Config.getEnv('USER')) + " " + Config.getEnv(
'USER') + '@' + user_realm
exit_code, stdout = Storm.runStormHdfsTopology(
TARGET_HIVE_STORM_JAR,
className,
args,
None,
logoutput=True,
inBackground=False,
useStandaloneCmd=useStandaloneCmd)
logger.info(exit_code)
ruAssert("Storm", exit_code == 0,
"[StormHiveSubmit] %s Failed" % (tcId))
def runas(cls,
user,
cmd,
cwd=None,
env=None,
logoutput=True,
runInBackground=False):
runCmd = Config.get('pig', 'PIG_CMD') + " " + cmd
# initialize env
if not env:
env = {}
# get kerberos ticket
if Hadoop.isSecure():
if user is None:
user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
if runInBackground:
return Machine.runinbackgroundAs(user, runCmd, cwd=cwd, env=env)
else:
return Machine.runas(user,
runCmd,
cwd=cwd,
env=env,
logoutput=logoutput)
def getHSIInstances(cls):
"""
Returns the list of active and passive hive server interactive instances.
"""
hsi_dict = {'active': [], 'passive': []}
zkhosts = Zookeeper.getZKHosts()
if Hadoop.isSecure():
zNode = 'hs2ActivePassiveHA-sasl'
else:
zNode = 'hs2ActivePassiveHA-unsecure'
exit_code, stdout = Zookeeper.runZKCli("ls /%s/instances" % zNode, server=zkhosts[0])
instances = stdout.split('\n')[-1]
logger.info(instances)
if instances == '[]':
return hsi_dict
for each_bracket in ['[', ']']:
instances = instances.replace(each_bracket, '')
instances_list = instances.split(', ')
for each_instance in instances_list:
exit_code, out = Zookeeper.runZKCli("get /%s/instances/%s" % (zNode, each_instance), server=zkhosts[0])
for line in out.split("\n"):
match = re.search(".*JSONServiceRecord.*", line)
if match:
json_data = line
break
instance_dict = ast.literal_eval(json_data)
instance_host = instance_dict['hive.server2.thrift.bind.host']
if instance_dict['internal'][0]['api'] == 'passiveEndpoint':
hsi_dict['passive'].append(instance_host)
else:
hsi_dict['active'].append(instance_host)
logger.info(hsi_dict)
return hsi_dict
def setup_storm_hdfs_topology(cls, useStandaloneCmd):
storm_version = Storm.getVersion(useStandaloneCmd=True)
try:
file_obj = open(HDFS_CONFIG_FILE, 'w')
if Hadoop.isSecure():
file_obj.write('hdfs.keytab.file: ' +
Machine.getHeadlessUserKeytab(
user=HADOOPQA_USER) + '\n')
file_obj.write('hdfs.kerberos.principal: ' +
Machine.get_user_principal(user=HADOOPQA_USER) +
'\n')
finally:
file_obj.close()
HDFS.createDirectory("/tmp/mySeqTopology", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/dest", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/dest2", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/foo", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/trident", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/trident-seq", HDFS_USER, "777", False)
Machine.copy(JAVA_HDFS_SRC_DIR,
LOCAL_HDFS_WORK_DIR,
user=None,
passwd=None)
if not Machine.isWindows():
(exit_code, _) = Maven.run('package',
cwd=LOCAL_HDFS_WORK_DIR,
env={
HADOOP_VERSION_MAVEN_PARAMETER:
HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER:
storm_version,
HADOOP_CONF_MAVEN_PARAMETER:
HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER:
HDFS_FILE,
HADOOP_CORE_MAVEN_PARAMETER:
HADOOP_CONF,
CORE_FILE_MAVEN_PARAMETER:
CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER:
Maven.getPublicRepoUrl()
})
else:
filepath = os.path.join(MOD_CONF_PATH, "core-site.xml")
(exit_code, _) = Maven.run(
'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s'
% (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER, storm_version,
HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE,
HADOOP_CORE_MAVEN_PARAMETER, HADOOP_CONF,
CORE_FILE_MAVEN_PARAMETER, CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()),
cwd=LOCAL_HDFS_WORK_DIR)
ruAssert("Storm", exit_code == 0,
"[StormHDFSSetup] maven package command failed")
def runOozieJobsCmdAs(cls,
user,
cmd,
cwd=None,
env=None,
logoutput=True,
retry=False,
num_of_retries=5,
wait=30,
oozie_server=None):
if not env:
env = {}
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
# if oozie server is not sent find it.
logger.info("OOZIE SERVER:%s" % oozie_server)
if not oozie_server:
oozie_server = cls.getOozieUrl()
logger.info("OOZIE SERVER:%s" % oozie_server)
env['JAVA_HOME'] = Config.get("machine", "JAVA_HOME")
if Machine.type() == 'Windows':
paramsList = cmd.split()
escapedCmd = ""
for param in paramsList:
if param[0] != '"' and param[-1] != '"':
escapedCmd = escapedCmd + "\"" + param + "\"" + " "
else:
escapedCmd = escapedCmd + param + " "
oozie_cmd = OOZIE_CMD + " jobs -oozie " + oozie_server + " " + escapedCmd
else:
oozie_cmd = OOZIE_CMD + " jobs -oozie " + oozie_server + " " + cmd
exit_code, output = Machine.runas(user,
oozie_cmd,
cwd=cwd,
env=env,
logoutput=logoutput)
count = 1
# Handle retires if the user selects
while retry and exit_code != 0 and count < num_of_retries:
exit_code, output = Machine.runas(user,
oozie_cmd,
cwd=cwd,
env=env,
logoutput=logoutput)
count += 1
time.sleep(wait)
return exit_code, output
def serverStop(cls, cwd=None, env=None, logoutput=True):
cmd = SQOOP2_SERVER + "stop" #Config.get('sqoop2', 'SQOOP2_CMD')
osenv = {"JAVA_HOME": Config.get('machine', 'JAVA_HOME')}
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
osenv['KRB5CCNAME'] = kerbTicket
user = None
if env:
for key, value in env.items():
osenv[key] = value
return Machine.run(cmd, cwd=cwd, env=osenv, logoutput=logoutput)
def runas(cls, user, cmd, cwd=None, env=None, logoutput=True, disableAuth=False):
if not env:
env = {}
# if disable auth is requsted set the env
# var to empty only for a secure cluster
if disableAuth and Hadoop.isSecure():
env['KRB5CCNAME'] = ''
# if disableAuth is false and we are running a secure cluster get
# the credentials
elif Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
mahout_cmd = Config.get('mahout', 'MAHOUT_CMD')
mahout_cmd += " " + cmd
osenv = {"JAVA_HOME": Config.get('machine', 'JAVA_HOME')}
if env:
for key, value in env.items():
osenv[key] = value
return Machine.runas(user, mahout_cmd, cwd=cwd, env=osenv, logoutput=logoutput)
def runas(cls, user, cmd, env=None, logoutput=True, disableAuth=False):
if not env:
env = {}
# if disable auth is requsted set the env
# var to empty only for a secure cluster
if disableAuth and Hadoop.isSecure():
env['KRB5CCNAME'] = ''
env = env.items() + ENVIRONMENT.items()
# if disableAuth is false and we are running a secure cluster get
# the credentials
elif Hadoop.isSecure():
if user is None:
user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
return Machine.runas(
user,
cmd,
env=dict(env.items() +
ENVIRONMENT.items() if env is not None else ENVIRONMENT),
logoutput=logoutput)
def getOozieJobList(cls, status):
'''
Returns an array of jobID
'''
jobArray = []
regex = '[\d]{7}-[\d]{15}-oozie-oozi-[W|B|C]'
for jobType in ('wf', 'coordinator', 'bundle'):
cmd = ' jobs -oozie=%s -filter status=%s -jobtype %s' % (
cls.getOozieUrl(), status, jobType)
if Hadoop.isSecure():
cmd = " -Doozie.auth.token.cache=false " + cmd
exit_code, stdout = cls.runas(cls._oozie_user, cmd)
jobArray += re.findall(regex, stdout)
return jobArray
def _extendedHiveserver2Url(cls, isHttpMode, isLdapMode, isDirectMode, useSSL):
hs2url = ''
if not isDirectMode and cls.isDynamicServiceDiscoveryOn():
zknamespace = cls.getZkNamespace()
hs2url += (";serviceDiscoveryMode=%s;zooKeeperNamespace=" % cls.getServiceDiscoveryMode()) + zknamespace
isLdapMode = isLdapMode or cls.getConfigValue("hive.server2.authentication", "NONE").lower() == "ldap"
if not isLdapMode and Hadoop.isSecure():
principal = cls.getConfigValue("hive.server2.authentication.kerberos.principal")
hs2url += ";principal=" + principal
if isHttpMode or cls.getConfigValue("hive.server2.transport.mode", "binary") == "http":
hs2url += ";transportMode=http;httpPath=cliservice"
if useSSL and cls.getConfigValue("hive.server2.use.SSL", defaultValue="false").lower() == "true":
trustStorePath = "/etc/security/serverKeys/hivetruststore.jks"
trustStorePwd = "changeit"
hs2url += ";ssl=true;sslTrustStore=%s;trustStorePassword=%s" % (trustStorePath, trustStorePwd)
return hs2url
def getClasspathForJdbcClient():
jdbc_standalone_jar = getStandaloneHiveJdbcJar()
assert jdbc_standalone_jar, "No JDBC standalone jar found"
classpath = [jdbc_standalone_jar]
hadoop_home = Config.get('hadoop', 'HADOOP_HOME')
if Hadoop.isHadoop2() and Machine.type() == 'Windows':
hadoop_home = os.path.join(hadoop_home, "share", "hadoop", "common")
hadoop_common_jar = util.findMatchingFiles(hadoop_home, "hadoop-common-*[!(tests)].jar", depth=1)
assert len(hadoop_common_jar) > 0, "No hadoop-common.jar found"
classpath.append(hadoop_common_jar[0])
if Hadoop.isSecure():
hadoop_auth_jar = util.findMatchingFiles(hadoop_home, "hadoop-auth-*[!(tests)].jar", depth=1)
assert len(hadoop_auth_jar) > 0, "No hadoop-auth.jar found"
classpath.append(hadoop_auth_jar[0])
classpath.append(Config.get('hadoop', 'HADOOP_CONF'))
return (os.pathsep).join(classpath)
def createClusterEntities(cls, colo, desc, name):
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
from beaver.component.hadoop import Hadoop, HDFS, YARN
write_endpoint = Hadoop.getFSDefaultValue()
webhdfs_scheme = 'webhdfs'
if HDFS.isHttpsEnabled():
webhdfs_scheme = 'swebhdfs'
read_endpoint = '%s://%s:%s' % (
webhdfs_scheme, write_endpoint.split('/')[2].split(':')[0],
HDFS.getNNWebPort())
execute_endpoint = YARN.getResourceManager()
falconNode = Falcon.get_falcon_server()
from beaver.component.oozie import Oozie
oozieUrl = Oozie.getOozieUrl()
entityText = "<?xml version=\"1.0\"?>" \
"<cluster colo=\"" + colo + "\" description=\"" + desc + "\" name=\"" + name + "\" " \
"xmlns=\"uri:falcon:cluster:0.1\"> " \
"<interfaces> " \
"<interface type=\"readonly\" endpoint=\""+read_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"write\" endpoint=\""+write_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"execute\" endpoint=\"" + execute_endpoint + "\" version=\"0.20.2\"/> " \
"<interface type=\"workflow\" endpoint=\"" + oozieUrl + "\" version=\"3.1\"/>" \
"<interface type=\"messaging\" endpoint=\"" \
"tcp://" + falconNode + ":61616?daemon=true\" version=\"5.1.6\"/>" \
"</interfaces>" \
"<locations>" \
"<location name=\"staging\" path=\"/apps/falcon/" + name + "/staging\" />" \
"<location name=\"temp\" path=\"/tmp\" />" \
"<location name=\"working\" path=\"/apps/falcon/" + name + "/working\" />" \
"</locations>" \
"<ACL owner=\"" + cls._job_user + "\" group=\"users\" permission=\"0755\"/>"
if Hadoop.isSecure():
realm = HDFS.getConfigValue(
'dfs.namenode.kerberos.principal').split('@')[1]
entityText += "<properties> <property name=\"dfs.namenode.kerberos.principal\" value=\"nn/_HOST@" + realm + "\"/> </properties>"
entityText += "</cluster>"
textFile = open(os.path.join(cls._local_workspace, name + ".xml"), "w")
textFile.write("%s" % entityText)
textFile.close()
return
def run(cls, cmd, cwd=None, env=None, logoutput=True, user=None):
# make sure maven is setup before its run
cls.setupAnt()
# initialize env
if not env:
env = {}
# if running a secure cluster get a kerb ticket
if Hadoop.isSecure():
env['KRB5CCNAME'] = Machine.getKerberosTicket(user)
env['JAVA_HOME'] = cls._java_home
env['ANT_HOME'] = cls._ant_home
run_cmd = "%s -Dbuild.compiler=javac1.7 %s" % (cls._ant_cmd, cmd)
exit_code, stdout = Machine.run(run_cmd, cwd=cwd, env=env, logoutput=logoutput)
return exit_code, stdout
def submit_storm_hbase_topology(cls, tcId, className, args,
useStandaloneCmd):
if Hadoop.isSecure():
keytab_file = Machine.getHeadlessUserKeytab(user=HADOOPQA_USER)
principal = Machine.get_user_principal(user=HADOOPQA_USER)
args = args + " %s %s" % (keytab_file, principal)
exit_code, stdout = Storm.runStormHdfsTopology(
TARGET_HBASE_STORM_JAR,
className,
args,
None,
logoutput=True,
inBackground=False,
useStandaloneCmd=useStandaloneCmd)
logger.info(exit_code)
ruAssert("Storm", exit_code == 0,
"[StormHBaseSubmit] %s Failed" % (tcId))
def runas(cls, user, cmd, cwd=None, env=None, logoutput=True):
hcat_cmd = cls._hcat_cmd + " " + cmd
# initialize env
if not env:
env = {}
# get kerberos ticket
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
return Machine.runas(user,
hcat_cmd,
cwd=cwd,
env=env,
logoutput=logoutput)
def getLLAPDaemonPidsHosts():
hosts = []
llapdaemon_pids = []
if not Machine.isHumboldt(): nodes = Hadoop.getAllNodes()
else: nodes = HDFS.getDatanodes()
for node in nodes:
pids = Machine.getProcessListRemote(
node, format="%U %p %P %a", filter="org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon", logoutput=True
)
if pids:
if Hadoop.isSecure():
pid = Machine.getPidFromString(pids[0], Config.get('hive', 'HIVE_USER'))
else:
pid = Machine.getPidFromString(pids[0], Config.get('hadoop', 'YARN_USER'))
llapdaemon_pids.append(pid)
hosts.append(node)
return llapdaemon_pids, hosts
def setup_phoenix(self):
from beaver.component.hadoop import Hadoop
PHOENIX_TEST_TABLE = 'basicTable'
try:
logger.info("### Phoenix setup starting ###")
from beaver.component.hbase import HBase
from beaver.component.phoenix import Phoenix
from beaver.component.xa import Xa
# servers = HBase.getNumOfRegionServers()
# if servers == -1:
# time.sleep(10)
# servers = HBase.getNumOfRegionServers()
#
# if servers == -1 or servers['running'] <= 0:
# #We restart all the regionServers
# HBase.startRegionServers(nodes=HBase.getRegionServers())
if Hadoop.isSecure() and not Xa.isArgus():
Phoenix.grantPermissionsToSystemTables()
#We create a table to read through the upgrade process
primaryKey = {'name': 'ID', 'type': 'BIGINT'}
columns = [{
'name': 'FirstName',
'type': 'VARCHAR(30)'
}, {
'name': 'SecondName',
'type': 'VARCHAR(30)'
}, {
'name': 'City',
'type': 'VARCHAR(30)'
}]
exit_code, stdout = Phoenix.createTable(PHOENIX_TEST_TABLE,
primaryKey, columns)
env = {}
env['JAVA_HOME'] = Config.get('machine', 'JAVA_HOME')
#We insert 10 rows into the table
for i in range(10):
Phoenix.runSQLLineCmds(
"UPSERT INTO %s VALUES (%s,'Name_%s','Surname_%s','City_%s')"
% (PHOENIX_TEST_TABLE, str(i), str(i), str(i), str(i)),
env)
except Exception as phoenix_exception:
logger.info("### Phoenix setup failed ###")
logger.info('Caused by: ' + str(phoenix_exception))
pass
def runCommandAs(cls, user, cmd, cwd=None, env=None, logoutput=True, stderr_as_stdout=True, background=False):
# initialize env
if not env:
env = {}
# get kerberos ticket
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
if background:
return Machine.runinbackgroundAs(user, cmd, cwd=cwd, env=env)
if stderr_as_stdout:
return Machine.runas(user, cmd, cwd=cwd, env=env, logoutput=logoutput)
else:
return Machine.runexas(user, cmd, cwd=cwd, env=env, logoutput=logoutput)
def start_LongRunning_HDFS_stream_job(cls,
inputDir,
num_executor,
mode="yarn-client",
inBackground=True,
clientfile=None):
'''
Start Spark-HDFS Streaming application
'''
className = "org.apache.spark.examples.streaming.HdfsWordCount"
if mode == "yarn-client" and not HDFS.isASV():
jars = Spark.getLzoJar()
else:
jars = None
if clientfile == None:
Local_clientlog = Spark.createTmpClientFile(className + "_" + mode)
else:
Local_clientlog = Spark.createTmpClientFile(clientfile)
arg = " %s 2>&1 | tee %s" % (inputDir, Local_clientlog)
if Hadoop.isSecure():
keytab = Machine.getHeadlessUserKeytab(
Config.get('hadoop', 'HADOOPQA_USER'))
principal = Machine.get_user_principal(
Config.get('hadoop', 'HADOOPQA_USER'))
else:
keytab = None
principal = None
Spark.submitSparkApplication(className,
mode,
arg,
jars=jars,
num_executor=num_executor,
inBackground=inBackground,
timeout=120,
keytab=keytab,
principal=principal)
f = open(Local_clientlog, "r")
stdout = f.read()
f.close()
appId = YARN.getApplicationIDFromStdout(stdout)
return appId, Local_clientlog
def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Setup for background long running job
:param runSmokeTestSetup: Runs smoke test setup if set to true
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][FLUME][BGJobSetup] Long running job setup for Flume component started"
)
from beaver.component.flume import Agent
global agent1
global agent2
if not os.path.exists(cls._local_work_dir):
os.mkdir(cls._local_work_dir)
shutil.copy(cls._flume_datagen_src, cls._local_work_dir)
agent1 = Agent(cls._local_work_dir)
agent2 = Agent(cls._local_work_dir)
for outdir in (cls._agent1_chkpt_dir, cls._agent1_data_dir,
cls._agent2_chkpt_dir, cls._agent2_data_dir):
os.mkdir(outdir)
logger.info("Preparing the Flume configs for long running test")
propertyMap = {}
namenode = Hadoop.getFSDefaultValue()
propertyMap['agent2.sinks.hdfsSink.hdfs.path'] = "%s%s" % (
namenode, cls._hdfs_test_dir)
if Hadoop.isSecure():
if Config.hasOption('machine', 'USER_REALM'):
user_realm = Config.get('machine', 'USER_REALM', '')
else:
nnKerbPrincipal = HDFS.getNameNodePrincipal(defaultValue='')
atloc = nnKerbPrincipal.find("@")
if atloc != -1:
user_realm = nnKerbPrincipal[atloc:]
if user_realm:
propertyMap[
'agent2.sinks.hdfsSink.hdfs.kerberosPrincipal'] = cls._test_user + '@' + user_realm
propertyMap[
'agent2.sinks.hdfsSink.hdfs.kerberosKeytab'] = Machine.getHeadlessUserKeytab(
cls._test_user)
util.writePropertiesToFile(
os.path.join(cls._flume_test_conf, 'longrunning.properties'),
cls._flume_test_src, propertyMap)
def runas(cls,
user,
cmd,
cwd=None,
env=None,
sqoopenv=None,
logoutput=True,
stderr_as_stdout=True):
sqoop_cmd = Config.get('sqoop', 'SQOOP_CMD')
sqoop_cmd += " " + cmd
osenv = {"JAVA_HOME": Config.get('machine', 'JAVA_HOME')}
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
osenv['KRB5CCNAME'] = kerbTicket
user = None
if env:
for key, value in env.items():
osenv[key] = value
if sqoopenv:
if Machine.type() == 'Windows':
sqoopenvcmd = "set SQOOP_CMD_OPTS=" + sqoopenv
sqoop_cmd = sqoopenvcmd + "&" + sqoop_cmd
else:
sqoopenvcmd = "export SQOOP_CMD_OPTS=" + "\"" + sqoopenv + "\""
sqoop_cmd = sqoopenvcmd + ";" + sqoop_cmd
if stderr_as_stdout:
return Machine.runas(user,
sqoop_cmd,
cwd=cwd,
env=osenv,
logoutput=logoutput)
else:
return Machine.runexas(user,
sqoop_cmd,
cwd=cwd,
env=osenv,
logoutput=logoutput)
def setup_storm_jdbc_topology(cls, useStandaloneCmd):
from beaver.dbutil import MySQL
MySQL.createUserWithAllPriveleges(STORM_JDBC_TEST_USER,
STORM_JDBC_TEST_PASSWD)
storm_version = Storm.getVersion(useStandaloneCmd=True)
try:
file_obj = open(HDFS_CONFIG_FILE, 'w')
if Hadoop.isSecure():
file_obj.write('hdfs.keytab.file: ' +
Machine.getHeadlessUserKeytab(
user=HADOOPQA_USER) + '\n')
file_obj.write('hdfs.kerberos.principal: ' +
Machine.get_user_principal(user=HADOOPQA_USER) +
'\n')
finally:
file_obj.close()
Machine.copy(JAVA_JDBC_SRC_DIR,
LOCAL_JDBC_WORK_DIR,
user=None,
passwd=None)
post_fenton_opt = " -DpostFenton=true" if Storm.isAfterFenton() else ""
package_cmd = 'package ' + post_fenton_opt
(exit_code, _) = Maven.run(package_cmd,
cwd=LOCAL_JDBC_WORK_DIR,
env={
HADOOP_VERSION_MAVEN_PARAMETER:
HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER:
storm_version,
HADOOP_CONF_MAVEN_PARAMETER:
HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER:
HDFS_FILE,
CORE_FILE_MAVEN_PARAMETER:
CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER:
Maven.getPublicRepoUrl()
})
ruAssert("Storm", exit_code == 0,
"[StormJDBCSetup] maven package command failed")
def upgrade_slave(cls, version, node, config=None):
'''
Upgrades Yarn slave services : NM
:param version: Version to be upgraded to
:param node: Slave Node
:param config: Config location
:return:
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Nodemanager Upgrade for %s started " % node)
logger.info("**** Beginning upgrade for nodemanager on %s ****" % node)
MAPRED.stopTasktrackers(nodes=[node])
from beaver.component.rollingupgrade.ruCommon import hdpSelect
hdpSelect.changeVersion("hadoop-yarn-nodemanager", version, node)
# BUG-27328
# make sure we set the linux container executor permissions
if Hadoop.isSecure() and not Machine.isWindows():
container_exec_cfg = os.path.join(
Config.get('hadoop', 'HADOOP_CONF'), 'container-executor.cfg')
container_exec = os.path.join(
Config.get('hadoop',
'YARN_HOME').replace("client", "nodemanager"),
'bin', 'container-executor')
cmd="chown root:hadoop %s %s ; chmod 400 %s ; chmod 6050 %s" % \
(container_exec_cfg, container_exec, container_exec_cfg, container_exec)
Machine.runas(Machine.getAdminUser(),
cmd,
host=node,
cwd=None,
env=None,
logoutput=True,
passwd=Machine.getAdminPasswd())
MAPRED.startTasktrackers(config, nodes=[node])
logger.info("**** Ending upgrade for nodemanager on %s ****" % node)
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Nodemanager Upgrade on %s Finished " % node)
def runas(cls,
user,
cmd,
cwd=None,
env=None,
logoutput=True,
base_cmd=OOZIE_CMD):
if not env:
env = {}
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
env['JAVA_HOME'] = Config.get("machine", "JAVA_HOME")
oozie_cmd = base_cmd + " " + cmd
return Machine.runas(user,
oozie_cmd,
cwd=cwd,
env=env,
logoutput=logoutput)
def runOozieAdminCmdAs(cls,
user,
cmd,
host=None,
cwd=None,
env=None,
logoutput=True):
if not env:
env = {}
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
if host is None:
oozie_server = cls.getOozieUrl()
else:
oozie_server = host
env['JAVA_HOME'] = Config.get("machine", "JAVA_HOME")
if Machine.type() == 'Windows':
paramsList = cmd.split()
escapedCmd = ""
for param in paramsList:
escapedCmd = escapedCmd + "\"" + param + "\"" + " "
oozie_cmd = OOZIE_CMD + " admin -oozie " + oozie_server + ":" + OOZIE_PORT + "/oozie " + escapedCmd
else:
oozie_cmd = OOZIE_CMD + " admin -oozie " + oozie_server + ":" + OOZIE_PORT + "/oozie " + cmd
exit_code, output = Machine.runas(user,
oozie_cmd,
cwd=cwd,
env=env,
logoutput=logoutput)
return exit_code, output
def deleteHDFSSnapshots(cls, endpoint, directory, _beacon_host=None):
if HDFS.isHAEnabled() and _beacon_host != None:
snapshotendpoint = "hdfs://" + HDFS.getActiveNN(
_beacon_host, "NAMENODE")
snapshot_names = BeaconHDFS.getSnapshotList(
snapshotendpoint, directory)
for snapshot_name in snapshot_names:
exit_code, stdout = HDFS.deleteSnapshot(snapshotendpoint +
directory,
snapshot_name,
user=hdfs_user)
#assert exit_code == 0
else:
snapshot_names = BeaconHDFS.getSnapshotList(endpoint, directory)
for snapshot_name in snapshot_names:
exit_code, stdout = HDFS.deleteSnapshot(endpoint + directory,
snapshot_name,
user=hdfs_user)
if _beacon_host is None:
exit_code, stdout = HDFS.disallowSnapshot(endpoint + directory,
user=hdfs_user)
assert exit_code == 0
else:
disallowsnapshot = " hdfs dfsadmin -disallowSnapshot " + endpoint + directory
if Hadoop.isSecure():
hdfsuser = Config.get("hadoop", 'HDFS_USER')
kinitCmd = "kinit -k -t " + Machine.getHeadlessUserKeytab(
hdfsuser) + " " + hdfsuser + "@" + Machine.get_user_realm(
)
exit_code, stdout = Machine.runas(user=hdfsuser,
cmd=kinitCmd,
host=_beacon_host)
exit_code, stdout = Machine.runas(user=hdfs_user,
cmd=disallowsnapshot,
host=_beacon_host)
assert exit_code == 0
def write_hbase_site(cls, config=None):
'''
Obtain hbase-site.xml from Slider HBase and write to modified config path
'''
global new_conf_path
from beaver.component.sliderhbase import SliderHBase
from beaver.component.slider import Slider
tmpHBaseConfFile = os.path.join(Machine.getTempDir(), "hbase-site.xml")
if Hadoop.isSecure():
hbasesite = Slider.registry(
"sliderhbase",
flags="--getconf hbase-site --out " + tmpHBaseConfFile,
format="xml",
user=cls.HBASE_USER,
userFlag=cls.HBASE_USER
)
else:
hbasesite = Slider.registry(
"sliderhbase",
flags="--getconf hbase-site --out " + tmpHBaseConfFile,
format="xml",
user=cls.HBASE_USER
)
propertyMap = {'hbase.tmp.dir': '/tmp/hbase-tmp'}
generated_hbase_conf = os.path.join(new_conf_path, "hbase-site.xml")
# QE-3108
# if the hbase app is created successfully, a /tmp/hbase-site.xml will be
# generated from the app. Otherwise if the /tmp/hbase-site.xml is missing
# it means the slider app creation fails
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
if os.path.isfile(generated_hbase_conf):
util.writePropertiesToConfigXMLFile(tmpHBaseConfFile, generated_hbase_conf, propertyMap)
else:
UpgradePerNode.reportProgress(
"[FAILED][Slider][background] Slider hbase background setup failed due to hbase-site.xml not generated"
)
def clientOperation(cls,
cmd,
user=None,
host=None,
cwd=None,
env=None,
logoutput=True):
tfile = tempfile.NamedTemporaryFile(delete=False)
if host is not None:
tfile.write("set server --host %s --port 12000\n" % host)
tfile.write(cmd)
tfile.close()
Machine.runas(
Machine.getAdminUser(), "chown -R %s %s" %
(config.Config.get('sqoop2', 'SQOOP2_USER'), tfile.name))
Machine.runas(Machine.getAdminUser(), "chmod 755 %s" % tfile.name)
cmd = SQOOP2_CLIENT + tfile.name
osenv = {"JAVA_HOME": Config.get('machine', 'JAVA_HOME')}
if Hadoop.isSecure():
if user is None: user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
osenv['KRB5CCNAME'] = kerbTicket
#user = None
if env:
for key, value in env.items():
osenv[key] = value
if user is not None:
return Machine.runas(user,
cmd,
cwd=cwd,
env=osenv,
logoutput=logoutput)
else:
return Machine.run(cmd, cwd=cwd, env=osenv, logoutput=logoutput)
