def setup(cls, S3_AWS_ACCESS_KEY=None, S3_AWS_SECRET=None):
Machine.installPackageWithPip(packages="awscli",
hosts=Hadoop.getAllNodes(),
logoutput=True)
aws_home = "/root/.aws"
if not os.path.exists(aws_home):
Machine.makedirs(ADMIN_USER, GATEWAY_NODE, "/root/.aws", ADMIN_PWD)
util.writeToFile("[default]\nregion = us-west-2\noutput=json",
os.path.join(ARTIFACTS_DIR, "config"))
if S3_AWS_ACCESS_KEY:
cls._aws_access_key_id = S3_AWS_ACCESS_KEY
else:
cls._aws_access_key_id = Config.get('machine',
'S3_AWS_ACCESS_KEY')
if S3_AWS_SECRET:
cls._aws_secret_access_key = S3_AWS_SECRET
else:
cls._aws_secret_access_key = Config.get(
'machine', 'S3_AWS_SECRET')
util.writeToFile(
"[default]\naws_access_key_id = %s\naws_secret_access_key = %s"
% (cls._aws_access_key_id, cls._aws_secret_access_key),
os.path.join(ARTIFACTS_DIR, "credentials"))
Machine.runas(
ADMIN_USER,
"chown %s '%s/config'" % (ADMIN_USER, ARTIFACTS_DIR),
GATEWAY_NODE, ADMIN_PWD)
Machine.runas(
ADMIN_USER,
"chown %s '%s/credentials'" % (ADMIN_USER, ARTIFACTS_DIR),
GATEWAY_NODE, ADMIN_PWD)
Machine.copy(os.path.join(ARTIFACTS_DIR, "config"), aws_home,
ADMIN_USER, ADMIN_PWD)
Machine.copy(os.path.join(ARTIFACTS_DIR, "credentials"), aws_home,
ADMIN_USER, ADMIN_PWD)
def _postStopAction(cls, service):
if service == 'hiveserver2':
logger.info("Hard kill Tez sessions")
yarn_user = YARN.getYarnUser()
apps = YARN.getApplicationIDList(state='NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING')
if len(apps) > 0:
for app in apps:
YARN.killApplicationAs(app, user=yarn_user)
time.sleep(5)
logger.info("Hard kill the HS2 application if still running")
admin_user = Machine.getAdminUser()
hosts = cls.getServiceHosts(service)
port = cls.getHiveserver2ThriftPort()
for host in hosts:
pid = Machine.getPIDByPort(port, host=host, user=admin_user)
if pid:
logger.info("Found process for '%s' with PID %d" % (service, pid))
Machine.killProcessRemote(pid, host=host, user=admin_user)
time.sleep(2)
logger.info("Hard Kill proc_llap daemon due to BUG-62657")
allnodes = util.getAllNodes() if Machine.isHumboldt() else Hadoop.getAllNodes()
for node in allnodes:
proc_llap_pids = Machine.getProcessListRemote(
node, format="%U %p %P %a", filter="proc_llap", logoutput=True
)
if len(proc_llap_pids) != 0:
proc_llap_pid = Machine.getPidFromString(proc_llap_pids[0], yarn_user)
if proc_llap_pid:
logger.info("Found proc_llap process with PID %d on %s" % (proc_llap_pid, node))
Machine.killProcessRemote(proc_llap_pid, host=node, user=admin_user)
time.sleep(2)
def getAllNodes(cls):
'''
Get all nodes according to the platform used
'''
if Machine.isHumboldt():
allNodes = util.getAllNodes()
else:
allNodes = Hadoop.getAllNodes()
return allNodes
def getLLAPDaemonPidsHosts():
hosts = []
llapdaemon_pids = []
if not Machine.isHumboldt(): nodes = Hadoop.getAllNodes()
else: nodes = HDFS.getDatanodes()
for node in nodes:
pids = Machine.getProcessListRemote(
node, format="%U %p %P %a", filter="org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon", logoutput=True
)
if pids:
if Hadoop.isSecure():
pid = Machine.getPidFromString(pids[0], Config.get('hive', 'HIVE_USER'))
else:
pid = Machine.getPidFromString(pids[0], Config.get('hadoop', 'YARN_USER'))
llapdaemon_pids.append(pid)
hosts.append(node)
return llapdaemon_pids, hosts
