def refreshYarnQueues(cls, host_ip, weburl=source_weburl, cluster=None):
hostname = Machine.runas('root', 'hostname', host_ip)
if cluster == None:
cluster = Ambari.getClusterName()
if weburl == None:
weburl = Ambari.getWebUrl()
else:
cluster = Ambari.getClusterName(weburl=weburl)
uri = "/api/v1/clusters/%s/requests" % (cluster)
data = '{"RequestInfo":{"context":"Refresh YARN Capacity Scheduler","command":"REFRESHQUEUES","parameters/forceRefreshConfigTags":"capacity-scheduler"},' \
'"Requests/resource_filters":[{"service_name":"YARN","component_name":"RESOURCEMANAGER","hosts":"' + hostname[1] + '"}]}'
response = Ambari.http_put_post_request(uri=uri,
data=data,
requestType="POST",
weburl=target_weburl)
assert response.status_code == 202, "Failed To refresh Yarn Queue: Response Code: " + str(
response.status_code) + "Response body: " + response._content
def removeYarnQueue(cls,
queueName="newQueue",
defaultConfig=None,
webURL=source_weburl,
cluster=None):
if cluster == None:
cluster = Ambari.getClusterName()
if webURL == None:
webURL = Ambari.getWebUrl()
else:
cluster = Ambari.getClusterName(weburl=webURL)
# Delete new config
deleteConfig = {}
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".acl_administer_jobs"] = "*"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".acl_submit_applications"] = "*"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".capacity"] = "0"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".maximum-capacity"] = "0"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".state"] = "RUNNING"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".user-limit-factor"] = "1"
Ambari.deleteConfig(type='capacity-scheduler',
config=deleteConfig,
webURL=webURL)
# Reset to default config
Ambari.setConfig(type='capacity-scheduler',
config=defaultConfig,
webURL=webURL)
Ambari.start_stop_service('YARN',
'INSTALLED',
waitForCompletion=True,
weburl=webURL)
logger.info("---- Done stopping YARN cluster")
Ambari.start_stop_service('YARN',
'STARTED',
waitForCompletion=True,
weburl=webURL)
time.sleep(30)
logger.info("---- Done starting YARN cluster")
def restartLLAP(cls):
cluster = Ambari.getClusterName()
data = '{"RequestInfo":{"context":"Restart LLAP","command":"RESTART_LLAP"},"Requests/resource_filters":[{"service_name":"HIVE","component_name":"HIVE_SERVER_INTERACTIVE","hosts":"%s"}]}' % (
cls.getHiveHost()
)
url = '/api/v1/clusters/' + cluster + '/requests'
r = Ambari.http_put_post_request(url, data, 'POST')
assert (r.status_code == 200 or r.status_code == 201 or r.status_code == 202
), "Failed to restart LLAP on host %s, status=%d" % (cls.getHiveHost(), r.status_code)
time.sleep(240)
def getClusterNameUrl(self):
self.logger.info(self.urlGetClusters)
hres, res = self.callPyCurlRequest(
url=self.urlGetClusters, data=None, method='get', usernamepassword=self.username_password
)
d = json.loads(res)
if Config.hasSection('hdc'):
proxy_enabled = Config.get('hdc', 'USE_CLI')
if proxy_enabled == 'yes':
return str(d['items'][0]['href'])
return Ambari.getWebUrl() + "/api/v1/clusters/" + Ambari.getClusterName()
def is_secure(cls):
url = "%s/api/v1/clusters/%s" % (Ambari.getWebUrl(), Ambari.getClusterName())
retcode = None
for i in range(10):
retcode, retdata, retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
try:
if jsoncontent['Clusters']['security_type'] == "KERBEROS":
return True
except:
pass
util.sleep(5)
return False
def get_ambari_cluster_name(cls, refresh=False):
if not cls._cluster_name or refresh:
cls._cluster_name = Ambari.getClusterName(is_hdp=False, is_enc=Ambari.is_ambari_encrypted())
return cls._cluster_name
def getComponentHosts(self, service, component):
clust_name = Ambari.getClusterName()
host_list = Ambari.getServiceHosts(service, component, cluster=clust_name, is_enc=Xa.isWireEncryptionOn())
hostList = ','.join([str(i) for i in host_list])
return hostList
def getClusterName(self):
return Ambari.getWebUrl() + "/api/v1/clusters/" + Ambari.getClusterName()
def get_ambari_cluster_name(cls, refresh=False):
if not cls._cluster_name or refresh:
cls._cluster_name = Ambari.getClusterName(is_hdp=False, is_enc=Nifi.get_ambari_server_ssl())
return cls._cluster_name
def get_ambari_cluster_name(cls):
return Ambari.getClusterName(is_hdp=False,
is_enc=cls.get_ambari_server_ssl())
CONF['KNOX_BASE_URL'] = "%s://%s:%s/gateway" % (
CONF['KNOX_PROTO'], CONF['KNOX_HOST'], CONF['KNOX_PORT'])
CONF['KNOX_TOPO_URL'] = "%s/%s" % (CONF['KNOX_BASE_URL'], CONF['KNOX_TOPO'])
CONF['KNOX_WEBHDFS_URL'] = "%s/%s/webhdfs/v1/" % (CONF['KNOX_BASE_URL'],
CONF['KNOX_TOPO'])
CONF['DIRECT_WEBHDFS_URL'] = "http://%s/webhdfs/v1/" % (
HDFS.getNamenodeHttpAddress())
CONF['RANGER_KNOX_POLICY'] = None
#CONF['SRC_DIR'] = os.path.join(Config.getEnv('WORKSPACE'), 'tests', 'knox', 'knox_2')
knox_host = CONF['KNOX_HOST']
if Machine.isOpenStack():
knox_host = knox_host + ".com"
KNOXSSO_PROVIDER_URL = "%s://%s:%s/gateway/knoxsso/api/v1/websso" % (
CONF['KNOX_PROTO'], knox_host, CONF['KNOX_PORT'])
CLUSTER_NAME = Ambari.getClusterName(is_enc=Hadoop.isEncrypted())
KNOX_TRUSTSTORE_PASSWORD = "******"
KNOX_KEYSTORE_PATH = "/usr/hdp/current/knox-server/data/security/keystores/"
KNOX_TOPOLOGY_DIR = "/etc/knox/conf/topologies/"
JAVA_HOME = Config.get("machine", "QA_CODE_JAVA_HOME")
_workspace = Config.getEnv('WORKSPACE')
_artifact = Config.getEnv('ARTIFACTS_DIR')
global apicorelib
DEPLOY_CODE_DIR = os.path.join(Config.getEnv('WORKSPACE'), '..',
'ambari_deploy')
uifrm_folder = "uifrm_old/uifrm"
amb_prop_file = os.path.join(DEPLOY_CODE_DIR, uifrm_folder,
'ambari.properties')
import logging
import time
from beaver.component.ambari import Ambari
from beaver.component.xa import Xa, XaPolicy
from beaver.config import Config
logger = logging.getLogger(__name__)
ARTIFACTS_DIR = Config.getEnv('ARTIFACTS_DIR')
_ambari_host_1 = Config.get("multicluster", "AMBARI_GATEWAY1")
_ambari_host_2 = Config.get("multicluster", "AMBARI_GATEWAY2")
source_weburl = Ambari.getWebUrl(hostname=_ambari_host_1)
target_weburl = Ambari.getWebUrl(hostname=_ambari_host_2)
source_datacenter = target_datacenter = "default"
primaryCluster = source_datacenter + "$" + Ambari.getClusterName(
weburl=source_weburl)
backupCluster = target_datacenter + "$" + Ambari.getClusterName(
weburl=target_weburl)
policy_prefix = 'pol_for_'
class BeaconRanger:
def __init__(self):
pass
policyIntervalChanged = False
policyActivationWaitTime = 30
policiesAddedBeforeTest = []
@classmethod
def changePolicyInterval(cls):
def get_ambari_cluster_name(cls):
return Ambari.getClusterName()