def getActiveNN(cls, host, component, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = cls.getweburl(host)
count = 5
length = 1
while count > 0 and length >= 1:
url = "%s/api/v1/clusters/%s/host_components?HostRoles/component_name=%s&metrics/dfs/FSNamesystem/HAState=active" % (
weburl, cluster, component)
auth = HTTPBasicAuth(Ambari.getAdminUsername(),
Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.get(url=url,
headers=head,
auth=auth,
verify=False)
assert response.status_code == 200, "Failed to get config versions."
json = response.json()
items = json['items']
length = len(items)
count = count - 1
time.sleep(10)
last = items[-1]
active_nn = last['HostRoles']['host_name']
return active_nn
def setConfig(cls, host, type, config, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = cls.getweburl(host)
tag = "version-%s" % str(uuid.uuid1())
data = {
'Clusters': {
'desired_config': {
'type': type,
'tag': tag,
'properties': config
}
}
}
json = jsonlib.dumps(data)
url = "%s/api/v1/clusters/%s" % (weburl, cluster)
auth = HTTPBasicAuth(Ambari.getAdminUsername(),
Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.put(url=url,
headers=head,
auth=auth,
data=json,
verify=False)
assert response.status_code == 200, "Failed to set config."
def stopComponent(cls, host, component, component_host, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = cls.getweburl(host)
url = "%s/api/v1/clusters/%s/hosts/%s/host_components/%s" % (
weburl, cluster, component_host, component)
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
data = '{ "HostRoles": { "state": "INSTALLED" } }'
auth = HTTPBasicAuth(Ambari.getAdminUsername(),
Ambari.getAdminPassword())
response = requests.put(url,
data=data,
headers=head,
auth=auth,
verify=False)
assert (response.status_code == 200 or response.status_code == 202
), "Failed to stop component %s on host %s, status=%d" % (
component, host, response.status_code)
if response.status_code is 202:
json_data = json.loads(response._content)
request_id = json_data['Requests']['id']
logger.info("Waiting for the service " + component + " to stop..")
while not cls.get_request_current_state(host, request_id,
cluster) == 'COMPLETED':
if cls.get_request_current_state(host, request_id,
cluster) == 'FAILED':
break
time.sleep(10)
def fix_qe_14910(cls):
# QE-14190: replace ${hdp.version} in mapreduce.application.classpath because it will case bad substitution error
config = Ambari.getConfig(type='mapred-site')
for key in config.keys():
value = config[key]
value = value.replace('${hdp.version}', '{{version}}')
config[key] = value
Ambari.setConfig(type='mapred-site', config=config)
Ambari.restart_services_with_stale_configs()
def get_request_current_state(cls, request_id, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = Ambari.getWebUrl()
url = "%s/api/v1/clusters/%s/requests/%s" % (weburl, cluster, request_id)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.get(url=url, headers=head, auth=auth, verify=False)
json_data = json.loads(response._content)
return json_data['Requests']['request_status']
def restoreConfig(cls, services=['metastore', 'hiveserver2'], restoreTez=False, waitForPortToOpen=True):
resetVersion = cls.getStartingAmbariServiceConfigVersion()
logger.info("Restoring Service Config Version to: %s" % resetVersion)
Ambari.resetConfig(cls._ambariServiceName, resetVersion)
if restoreTez and not cls.isHive2():
Ambari.resetConfig(cls._ambariTezServiceName, cls._ambariTezServiceConfigVersion)
cls._modifiedConfigs.clear()
cls._hiveServer2Port = None
if len(services) > 0:
cls.restartServices(services=services, waitForPortToOpen=waitForPortToOpen)
def stopService(cls, services=['metastore', 'hiveserver2']):
for service in services:
hosts = cls.getServiceHosts(service)
for host in hosts:
Ambari.stopComponent(
host, cls._compSvcMap[service], waitForCompletion=True, timeout=cls._compSvcTimeouts[service]
)
cls._postStopAction(service)
# Workaround for BUG-73424
time.sleep(90)
def restartLLAP(cls):
cluster = Ambari.getClusterName()
data = '{"RequestInfo":{"context":"Restart LLAP","command":"RESTART_LLAP"},"Requests/resource_filters":[{"service_name":"HIVE","component_name":"HIVE_SERVER_INTERACTIVE","hosts":"%s"}]}' % (
cls.getHiveHost()
)
url = '/api/v1/clusters/' + cluster + '/requests'
r = Ambari.http_put_post_request(url, data, 'POST')
assert (r.status_code == 200 or r.status_code == 201 or r.status_code == 202
), "Failed to restart LLAP on host %s, status=%d" % (cls.getHiveHost(), r.status_code)
time.sleep(240)
def start_stop_service(cls, service, state, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = Ambari.getWebUrl()
url = "%s/api/v1/clusters/%s/services/%s" % (weburl, cluster, service)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
data = '{"RequestInfo": {"context": "DRUID API ' + state + ' ' + service + '"}, "Body" : {"ServiceInfo": {"state": "' + state + '"}}}'
response = requests.put(url, data=data, headers=head, auth=auth, verify=False)
assert ( response.status_code == 200 or response.status_code == 202 ),\
"Failed to start/stop service %s , status=%d" % ( service, response.status_code )
return response
def changePolicyInterval(cls):
if not BeaconRanger.policyIntervalChanged:
interval_for_source_cls = Ambari.getConfig(
'ranger-hive-security', webURL=source_weburl
)['ranger.plugin.hive.policy.pollIntervalMs']
interval_for_target_cls = Ambari.getConfig(
'ranger-hive-security', webURL=target_weburl
)['ranger.plugin.hive.policy.pollIntervalMs']
if not interval_for_source_cls == "5000":
Xa.changePolicyInterval("HIVE", "5000", webURL=source_weburl)
if not interval_for_target_cls == "5000":
Xa.changePolicyInterval("HIVE", "5000", webURL=target_weburl)
BeaconRanger.policyIntervalChanged = True
BeaconRanger.policyActivationWaitTime = 6
def restart_all_services(cls, host, type, cluster=None):
if cluster == None:
cluster = 'cl1'
weburl = cls.getweburl(host)
url = "%s/api/v1/clusters/%s/services" % (weburl, cluster)
auth = HTTPBasicAuth(Ambari.getAdminUsername(),
Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.get(url=url, headers=head, auth=auth, verify=False)
json_data = json.loads(response._content)
for i in range(0, len(json_data['items'])):
service = json_data['items'][i]['ServiceInfo']['service_name']
logger.info("Restarting service : %s " % service)
cls.restart_service(service, host)
def getDruidComponentHost(cls, component, service='DRUID', cluster=None):
if cluster == None:
cluster = 'cl1'
host_names = []
url = "%s/api/v1/clusters/%s/services/%s/components/%s" % (Ambari.getWebUrl(), cluster, service, component)
auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword())
head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'}
response = requests.get(url=url, headers=head, auth=auth, verify=False)
assert response.status_code == 200, "Failed to get config versions."
json = response.json()
host_components = json['host_components']
for i in range(0, len(host_components)):
host_names.append(host_components[i]['HostRoles']['host_name'])
return host_names
def checkKafkaStatusOnMetronHost(cls, metron_host):
kafka_status = None
url = Ambari.getWebUrl(
is_hdp=False
) + "/api/v1/clusters/%s/hosts/%s/host_components/%s" % (
cls.get_ambari_cluster_name(), metron_host, cls.kafka_broker)
retcode, retdata, _retheaders = Ambari.performRESTCall(url)
if retcode == 200:
jsoncontent = util.getJSON(retdata)
try:
kafka_status = jsoncontent['HostRoles']['state']
except Exception:
logger.error('Kafka broker is not running on the Metron host')
kafka_status = None
return kafka_status
def isSecuredCluster(cls):
'''
Return whether the cluster is secured or not
:return: boolean
'''
cls._is_secure = Ambari.isSecure()
return cls._is_secure
def getHiveLogDir(cls, logoutput=False):
if cls._hiveLogDir is None:
properties = Ambari.getConfig('hive-env')
cls._hiveLogDir = properties['hive_log_dir']
if logoutput:
logger.info("Hive.getHiveLogDir returns %s" % cls._hiveLogDir)
return cls._hiveLogDir
def verify_Policy_Exists_after_replication(
cls,
servicetype,
verify_from_cluster=source_weburl,
custer_to_verify=target_weburl,
database=None,
path=None,
NoPolicyInTarget=False,
expectedDesc="created by beacon while importing from " +
primaryCluster,
preDenyPolicyStr=primaryCluster + "_beacon deny policy for "):
if Xa.isArgusInstalled() is True:
serviceName = "hadoop" if servicetype == "hdfs" else servicetype
serviceNameOfverify_from_cluster = \
Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True,
ambariWeburl=verify_from_cluster)[0]['name']
serviceNameOfverify_to_cluster = \
Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True,
ambariWeburl=custer_to_verify)[0]['name']
logger.info("verifying if policy exist in target cluster")
policies_in_source_Cluster = Xa.getPoliciesForResources(
servicetype,
serviceName=serviceNameOfverify_from_cluster,
ambariWeburl=verify_from_cluster,
database=database,
path=path)
policies_in_target_Cluster = Xa.getPoliciesForResources(
servicetype,
serviceName=serviceNameOfverify_to_cluster,
ambariWeburl=custer_to_verify,
database=database,
path=path)
if NoPolicyInTarget == False:
assert len(policies_in_target_Cluster
) != 0, "make sure policies were imported"
BeaconRanger.setIdOfAllPolicyToZero(
policies_in_source_Cluster, policies_in_target_Cluster,
expectedDesc)
logger.info("set of policies in target cluster: " +
str(policies_in_target_Cluster["policies"]))
for policy in policies_in_source_Cluster["policies"]:
logger.info("policy is " + str(policy))
assert policy in policies_in_target_Cluster["policies"]
logger.info(
"all policies are verified!! now will check for deny policy if it is true"
)
isDenyPolicyTrue = Ambari.getConfig(
'beacon-security-site',
webURL=source_weburl)['beacon.ranger.plugin.create.denypolicy']
all_policies_in_target_Cluster = Xa.getPolicy_api_v2(
servicetype, weburl=target_weburl)
if isDenyPolicyTrue == 'true':
dataset = path if servicetype == "hdfs" else database
BeaconRanger.denyPolicyValidation(
servicetype, dataset, all_policies_in_target_Cluster,
preDenyPolicyStr)
else:
assert len(policies_in_target_Cluster) == len(
policies_in_source_Cluster)
def print_environment_details(self):
UpgradeLogger.reportProgress("=====Environment Details=====\n ", True)
is_hdp = self.STACK_TYPE.lower() == "hdp"
UpgradeLogger.reportProgress(
"Ambari URL : " + Ambari.getWebUrl(is_hdp=is_hdp), True)
UpgradeLogger.reportProgress("Ambari OS : " + Ambari.getOS(), True)
UpgradeLogger.reportProgress(
"Stack Type : " + Config.get('ambari', 'STACK_TYPE'), True)
UpgradeLogger.reportProgress(
"Ambari DB : " + Config.get('ambari', 'AMBARI_DB'), True)
UpgradeLogger.reportProgress(
"Kerberos : " + Config.get('machine', 'IS_SECURE'), True)
UpgradeLogger.reportProgress(
"HA : " + Config.get('ambari', 'IS_HA_TEST'), True)
UpgradeLogger.reportProgress(
"Wire Encryption : " + Config.get('ambari', 'WIRE_ENCRYPTION'),
True)
def getBaseUrl(self):
from beaver.component.ambari import Ambari
GRAFANA_HOST = Ambari.getHostsForComponent('METRICS_GRAFANA')[0]
if Hadoop.isEncrypted() or Machine.isHumboldt():
GRAFANA_URL = "https://%s:3000/dashboard/db/hbase-tuning" % (GRAFANA_HOST)
else:
GRAFANA_URL = "http://%s:3000/dashboard/db/hbase-tuning" % (GRAFANA_HOST)
self.base_url = GRAFANA_URL
return self.base_url
