def get_client_properties(self):
from beaver.component.kafka import Kafka
from beaver.component.ambari import Ambari
Kafka.alterUser(
userName=self.client_username,
config="'SCRAM-SHA-256=[iterations=8192,password=%s],"
"SCRAM-SHA-512=[password=%s]'" % (self.client_password, self.client_password)
)
is_ambari_enc = Ambari.is_ambari_encrypted()
kafka_jaas_config = Ambari.getConfig(
"kafka_jaas_conf", webURL=Ambari.getWebUrl(is_hdp=False, is_enc=is_ambari_enc)
)
replacement_jaas_entry = self.jaas_entry
if not Kafka._isSecure:
replacement_jaas_entry = "\nKafkaServer {%s\n};" % self.jaas_entry
if self.to_be_replaced + replacement_jaas_entry not in kafka_jaas_config['content']:
print "old : %s" % kafka_jaas_config['content']
kafka_jaas_config['content'] = kafka_jaas_config['content'].replace(
self.to_be_replaced, self.to_be_replaced + replacement_jaas_entry
)
print "new : %s" % kafka_jaas_config['content']
Ambari.setConfig(
"kafka_jaas_conf", kafka_jaas_config, webURL=Ambari.getWebUrl(is_hdp=False, is_enc=is_ambari_enc)
)
Ambari.restart_services_with_stale_configs()
time.sleep(20)
return {'sasl.jaas.config': self.jaas_entry.replace("\n", " "), 'sasl.mechanism': 'SCRAM-SHA-256'}
def get_streamline_service_status(cls, cluster_name):
service_state = None
url = Ambari.getWebUrl(is_hdp=False) + "/api/v1/clusters/%s/services/STREAMLINE" % cluster_name
retcode, retdata, retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
service_state = jsoncontent["ServiceInfo"]["state"]
return service_state
def is_smm_service_installed(cls, cluster_name):
url = Ambari.getWebUrl(
is_hdp=False, is_enc=Ambari.is_ambari_encrypted()
) + "/api/v1/clusters/%s/services/STREAMSMSGMGR" % cluster_name
retcode, retdata, retheaders = Ambari.performRESTCall(url)
if retcode == 404:
return False
return True
def get_smm_service_status(cls, cluster_name):
service_state = None
url = Ambari.getWebUrl(
is_hdp=False, is_enc=Ambari.is_ambari_encrypted()
) + "/api/v1/clusters/%s/services/STREAMSMSGMGR" % cluster_name
retcode, retdata, retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
service_state = jsoncontent["ServiceInfo"]["state"]
return service_state
def get_request_current_state(cls, request_id, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = Ambari.getWebUrl()
url = "%s/api/v1/clusters/%s/requests/%s" % (weburl, cluster, request_id)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.get(url=url, headers=head, auth=auth, verify=False)
json_data = json.loads(response._content)
return json_data['Requests']['request_status']
def get_ambari_url(self):
'''
Method to get ambari url
:return:
'''
ambari_url = Ambari.getWebUrl()
host = socket.gethostbyname(socket.gethostname())
host_name = socket.gethostname()
if ".hwx.site" not in host_name:
host_name = host_name + ".hwx.site"
ambari_url = ambari_url.replace(host, host_name)
return ambari_url
def getClusterNameUrl(self):
self.logger.info(self.urlGetClusters)
hres, res = self.callPyCurlRequest(
url=self.urlGetClusters, data=None, method='get', usernamepassword=self.username_password
)
d = json.loads(res)
if Config.hasSection('hdc'):
proxy_enabled = Config.get('hdc', 'USE_CLI')
if proxy_enabled == 'yes':
return str(d['items'][0]['href'])
return Ambari.getWebUrl() + "/api/v1/clusters/" + Ambari.getClusterName()
def start_stop_service(cls, service, state, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = Ambari.getWebUrl()
url = "%s/api/v1/clusters/%s/services/%s" % (weburl, cluster, service)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
data = '{"RequestInfo": {"context": "DRUID API ' + state + ' ' + service + '"}, "Body" : {"ServiceInfo": {"state": "' + state + '"}}}'
response = requests.put(url, data=data, headers=head, auth=auth, verify=False)
assert ( response.status_code == 200 or response.status_code == 202 ),\
"Failed to start/stop service %s , status=%d" % ( service, response.status_code )
return response
def setConfig(cls, type, config, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = Ambari.getWebUrl()
tag = "version-%s" % str(uuid.uuid1())
data = {'Clusters': {'desired_config': {'type': type, 'tag': tag, 'properties': config}}}
json = jsonlib.dumps(data)
url = "%s/api/v1/clusters/%s" % (weburl, cluster)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.put(url=url, headers=head, auth=auth, data=json, verify=False)
assert response.status_code == 200, "Failed to set config."
def getDruidComponentHost(cls, component, service='DRUID', cluster=None):
if cluster == None:
cluster = 'cl1'
host_names = []
url = "%s/api/v1/clusters/%s/services/%s/components/%s" % (Ambari.getWebUrl(), cluster, service, component)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.get(url=url, headers=head, auth=auth, verify=False)
assert response.status_code == 200, "Failed to get config versions."
json = response.json()
host_components = json['host_components']
for i in range(0, len(host_components)):
host_names.append(host_components[i]['HostRoles']['host_name'])
return host_names
def is_secure(cls):
url = "%s/api/v1/clusters/%s" % (Ambari.getWebUrl(), Ambari.getClusterName())
retcode = None
for i in range(10):
retcode, retdata, retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
try:
if jsoncontent['Clusters']['security_type'] == "KERBEROS":
return True
except:
pass
util.sleep(5)
return False
def checkKafkaStatusOnMetronHost(cls, metron_host):
kafka_status = None
url = Ambari.getWebUrl(
is_hdp=False
) + "/api/v1/clusters/%s/hosts/%s/host_components/%s" % (
cls.get_ambari_cluster_name(), metron_host, cls.kafka_broker)
retcode, retdata, _retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
try:
kafka_status = jsoncontent['HostRoles']['state']
except Exception:
logger.error('Kafka broker is not running on the Metron host')
kafka_status = None
return kafka_status
def refreshYarnQueues(cls, host_ip, weburl=source_weburl, cluster=None):
hostname = Machine.runas('root', 'hostname', host_ip)
if cluster == None:
cluster = Ambari.getClusterName()
if weburl == None:
weburl = Ambari.getWebUrl()
else:
cluster = Ambari.getClusterName(weburl=weburl)
uri = "/api/v1/clusters/%s/requests" % (cluster)
data = '{"RequestInfo":{"context":"Refresh YARN Capacity Scheduler","command":"REFRESHQUEUES","parameters/forceRefreshConfigTags":"capacity-scheduler"},' \
'"Requests/resource_filters":[{"service_name":"YARN","component_name":"RESOURCEMANAGER","hosts":"' + hostname[1] + '"}]}'
response = Ambari.http_put_post_request(uri=uri,
data=data,
requestType="POST",
weburl=target_weburl)
assert response.status_code == 202, "Failed To refresh Yarn Queue: Response Code: " + str(
response.status_code) + "Response body: " + response._content
def print_environment_details(self):
UpgradeLogger.reportProgress("=====Environment Details=====\n ", True)
is_hdp = self.STACK_TYPE.lower() == "hdp"
UpgradeLogger.reportProgress(
"Ambari URL : " + Ambari.getWebUrl(is_hdp=is_hdp), True)
UpgradeLogger.reportProgress("Ambari OS : " + Ambari.getOS(), True)
UpgradeLogger.reportProgress(
"Stack Type : " + Config.get('ambari', 'STACK_TYPE'), True)
UpgradeLogger.reportProgress(
"Ambari DB : " + Config.get('ambari', 'AMBARI_DB'), True)
UpgradeLogger.reportProgress(
"Kerberos : " + Config.get('machine', 'IS_SECURE'), True)
UpgradeLogger.reportProgress(
"HA : " + Config.get('ambari', 'IS_HA_TEST'), True)
UpgradeLogger.reportProgress(
"Wire Encryption : " + Config.get('ambari', 'WIRE_ENCRYPTION'),
True)
def stopComponent(cls, component, component_host, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = Ambari.getWebUrl()
url = "%s/api/v1/clusters/%s/hosts/%s/host_components/%s" % (weburl, cluster, component_host, component)
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
data = '{ "HostRoles": { "state": "INSTALLED" } }'
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
response = requests.put(url, data=data, headers=head, auth=auth, verify=False)
assert (response.status_code == 200 or response.status_code == 202
), "Failed to stop component %s , status=%d" % (component, response.status_code)
if response.status_code is 202:
json_data = json.loads(response._content)
request_id = json_data['Requests']['id']
logger.info("Waiting for the service " + component + " to stop..")
while not cls.get_request_current_state(request_id) == 'COMPLETED':
if cls.get_request_current_state(request_id) == 'FAILED':
break
time.sleep(10)
def removeYarnQueue(cls,
queueName="newQueue",
defaultConfig=None,
webURL=source_weburl,
cluster=None):
if cluster == None:
cluster = Ambari.getClusterName()
if webURL == None:
webURL = Ambari.getWebUrl()
else:
cluster = Ambari.getClusterName(weburl=webURL)
# Delete new config
deleteConfig = {}
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".acl_administer_jobs"] = "*"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".acl_submit_applications"] = "*"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".capacity"] = "0"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".maximum-capacity"] = "0"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".state"] = "RUNNING"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".user-limit-factor"] = "1"
Ambari.deleteConfig(type='capacity-scheduler',
config=deleteConfig,
webURL=webURL)
# Reset to default config
Ambari.setConfig(type='capacity-scheduler',
config=defaultConfig,
webURL=webURL)
Ambari.start_stop_service('YARN',
'INSTALLED',
waitForCompletion=True,
weburl=webURL)
logger.info("---- Done stopping YARN cluster")
Ambari.start_stop_service('YARN',
'STARTED',
waitForCompletion=True,
weburl=webURL)
time.sleep(30)
logger.info("---- Done starting YARN cluster")
def getPhoenixQueryServers(cls, host=None, reRunStatus=False):
if host:
phoenixqueryservers = Ambari.getHostsForComponent(
cls._ambariComponentMapping.get('phoenix_query_server'),
weburl=Ambari.getWebUrl(hostname=host))
phoenixqueryservers = [
pqs.encode('ascii') for pqs in phoenixqueryservers
]
logger.info("---------- phoenixqueryservers: %s",
phoenixqueryservers)
return phoenixqueryservers
if (not cls._phoenixqueryservers) or reRunStatus:
cls._phoenixqueryservers = Ambari.getHostsForComponent(
cls._ambariComponentMapping.get('phoenix_query_server'))
cls._phoenixqueryservers = [
pqs.encode('ascii') for pqs in cls._phoenixqueryservers
]
logger.info("---------- cls._phoenixqueryservers: %s",
cls._phoenixqueryservers)
return cls._phoenixqueryservers
def getConfig(cls, type, property_name, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = Ambari.getWebUrl()
url = "%s/api/v1/clusters/%s/configurations?type=%s" % (weburl, cluster, type)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.get(url=url, headers=head, auth=auth, verify=False)
assert response.status_code == 200, "Failed to get config versions."
json = response.json()
items = json['items']
last = items[len(items) - 1]
tag = last['tag']
url = "%s/api/v1/clusters/%s/configurations?type=%s&tag=%s" % (weburl, cluster, type, tag)
response = requests.get(url=url, headers=head, auth=auth, verify=False)
assert response.status_code == 200, "Failed to get config."
json = response.json()
items = json['items']
last = items[len(items) - 1]
config = last['properties'][property_name]
return config
def getMetronRestProperty(cls, param_name):
'''
Return the property value for given Metron Rest property from Ambari config using REST API
:return: List
'''
metron_rest_prop = None
url = Ambari.getWebUrl(
is_hdp=False
) + "/api/v1/clusters/%s/configurations?type=%s&tag=%s" % (
cls.get_ambari_cluster_name(), cls.metron_rest_config_type,
cls.metron_config_tag)
retcode, retdata, _retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
try:
metron_rest_prop = [
hc['properties'][param_name] for hc in jsoncontent['items']
]
except Exception:
logger.error('Metron rest property %s variable is not set',
param_name)
metron_rest_prop = None
return metron_rest_prop
def getMetronHome(cls):
'''
Return the path to Metron Home from Ambari config using REST API
:return: String
'''
metron_home = None
url = Ambari.getWebUrl(
is_hdp=False
) + "/api/v1/clusters/%s/configurations?type=%s&tag=%s" % (
cls.get_ambari_cluster_name(), cls.metron_config_type,
cls.metron_config_tag)
retcode, retdata, _retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
try:
metron_home = [
hc['properties']['metron_home']
for hc in jsoncontent['items']
]
except Exception:
logger.error('Metron Home variable is not set')
metron_home = None
return metron_home
#
import logging, requests, json, random, time
from beaver.config import Config
import beaver.component.dataplane.coreApi.DP_CONSTANTS as DP_CONSTANTS
from beaver.component.ambari import Ambari
logger = logging.getLogger(__name__)
DP_IP_ADDR = Config.get("dataplane", "DP_HOST_ADDR")
DP_HTTP_URL = ("https://%s/" % DP_IP_ADDR)
DP_KNOX_URL = (
"https://%s:8443/gateway/knoxsso/api/v1/websso?originalUrl=https://%s/" %
(DP_IP_ADDR, DP_IP_ADDR))
TEST_AMBARI_HOST1 = Config.get("multicluster", "AMBARI_GATEWAY1")
TEST_AMBARI_HOST2 = Config.get("multicluster", "AMBARI_GATEWAY2")
TEST_AMBARI_WEB_URL1 = Ambari.getWebUrl(hostname=TEST_AMBARI_HOST1)
TEST_AMBARI_WEB_URL2 = Ambari.getWebUrl(hostname=TEST_AMBARI_HOST2)
class DP_Api(object):
def __init__(self):
self.session = requests.session()
self.session.headers['Content-Type'] = 'application/json'
self.session.verify = False
def getSessionCookies(self):
"""
:return:
"""
return self.session.cookies
from beaver.component.ambari import Ambari
import logging, time
from beaver.machine import Machine
from beaver.config import Config
from beaver.component.beacon.beacon import Beacon
logger = logging.getLogger(__name__)
_ambari_host_1 = Config.get("multicluster", "AMBARI_GATEWAY1")
_ambari_host_2 = Config.get("multicluster", "AMBARI_GATEWAY2")
source_weburl = Ambari.getWebUrl(hostname=_ambari_host_1)
target_weburl = Ambari.getWebUrl(hostname=_ambari_host_2)
class BeaconYarn:
@classmethod
def addYarnQueue(cls,
queueName="newQueue",
capacity=26,
webURL=source_weburl):
defaultYarnQueueConfig = Ambari.getConfig(service='YARN',
type='capacity-scheduler',
webURL=webURL)
updatedYarnQueueConfig = defaultYarnQueueConfig.copy()
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.queues"] = defaultYarnQueueConfig[
"yarn.scheduler.capacity.root.queues"] + "," + queueName
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.default.capacity"] = str(
def getClusterName(self):
return Ambari.getWebUrl() + "/api/v1/clusters/" + Ambari.getClusterName()
def get_ambari_url(cls):
if not cls._ambari_url:
is_ambari_encrypted = Ambari.is_ambari_encrypted()
cls._ambari_url = Ambari.getWebUrl(is_enc=is_ambari_encrypted)
logger.info("ambari url is: %s" % cls._ambari_url)
return cls._ambari_url
_workspace = Config.getEnv('WORKSPACE')
_artifact = Config.getEnv('ARTIFACTS_DIR')
global apicorelib
DEPLOY_CODE_DIR = os.path.join(Config.getEnv('WORKSPACE'), '..',
'ambari_deploy')
uifrm_folder = "uifrm_old/uifrm"
amb_prop_file = os.path.join(DEPLOY_CODE_DIR, uifrm_folder,
'ambari.properties')
if not os.path.isfile(amb_prop_file):
uifrm_folder = "uifrm"
amb_prop_file = os.path.join(DEPLOY_CODE_DIR, uifrm_folder)
apicorelib = APICoreLib(amb_prop_file)
if Hadoop.isEncrypted():
ambari_url = Ambari.getWebUrl(is_enc=True, hostname=CONF['AMBARI_HOST'])
else:
ambari_url = Ambari.getWebUrl(is_enc=False, hostname=CONF['AMBARI_HOST'])
@TaskReporter.report_test()
def setup_for_ranger():
global ranger_orig_url, admin_prop_loc
if (isRangerInstalled()):
CONF['XA_ADMIN_HOST'] = Ambari.getConfig(
'admin-properties',
webURL=ambari_url)['policymgr_external_url'].split('//',
1)[1].split(
':', 1)[0]
if Hadoop.isEncrypted():
CONF['XA_ADMIN_PORT'] = Ambari.getConfig(