def getActiveNN(cls, host, component, cluster=None): if cluster == None: cluster = 'cl1' weburl = cls.getweburl(host) count = 5 length = 1 while count > 0 and length >= 1: url = "%s/api/v1/clusters/%s/host_components?HostRoles/component_name=%s&metrics/dfs/FSNamesystem/HAState=active" % ( weburl, cluster, component) auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword()) head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'} response = requests.get(url=url, headers=head, auth=auth, verify=False) assert response.status_code == 200, "Failed to get config versions." json = response.json() items = json['items'] length = len(items) count = count - 1 time.sleep(10) last = items[-1] active_nn = last['HostRoles']['host_name'] return active_nn
def setConfig(cls, host, type, config, cluster=None): if cluster == None: cluster = 'cl1' weburl = cls.getweburl(host) tag = "version-%s" % str(uuid.uuid1()) data = { 'Clusters': { 'desired_config': { 'type': type, 'tag': tag, 'properties': config } } } json = jsonlib.dumps(data) url = "%s/api/v1/clusters/%s" % (weburl, cluster) auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword()) head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'} response = requests.put(url=url, headers=head, auth=auth, data=json, verify=False) assert response.status_code == 200, "Failed to set config."
def stopComponent(cls, host, component, component_host, cluster=None): if cluster == None: cluster = 'cl1' weburl = cls.getweburl(host) url = "%s/api/v1/clusters/%s/hosts/%s/host_components/%s" % ( weburl, cluster, component_host, component) head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'} data = '{ "HostRoles": { "state": "INSTALLED" } }' auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword()) response = requests.put(url, data=data, headers=head, auth=auth, verify=False) assert (response.status_code == 200 or response.status_code == 202 ), "Failed to stop component %s on host %s, status=%d" % ( component, host, response.status_code) if response.status_code is 202: json_data = json.loads(response._content) request_id = json_data['Requests']['id'] logger.info("Waiting for the service " + component + " to stop..") while not cls.get_request_current_state(host, request_id, cluster) == 'COMPLETED': if cls.get_request_current_state(host, request_id, cluster) == 'FAILED': break time.sleep(10)
def fix_qe_14910(cls): # QE-14190: replace ${hdp.version} in mapreduce.application.classpath because it will case bad substitution error config = Ambari.getConfig(type='mapred-site') for key in config.keys(): value = config[key] value = value.replace('${hdp.version}', '{{version}}') config[key] = value Ambari.setConfig(type='mapred-site', config=config) Ambari.restart_services_with_stale_configs()
def get_request_current_state(cls, request_id, cluster=None): if cluster == None: cluster = 'cl1' weburl = Ambari.getWebUrl() url = "%s/api/v1/clusters/%s/requests/%s" % (weburl, cluster, request_id) auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword()) head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'} response = requests.get(url=url, headers=head, auth=auth, verify=False) json_data = json.loads(response._content) return json_data['Requests']['request_status']
def restoreConfig(cls, services=['metastore', 'hiveserver2'], restoreTez=False, waitForPortToOpen=True): resetVersion = cls.getStartingAmbariServiceConfigVersion() logger.info("Restoring Service Config Version to: %s" % resetVersion) Ambari.resetConfig(cls._ambariServiceName, resetVersion) if restoreTez and not cls.isHive2(): Ambari.resetConfig(cls._ambariTezServiceName, cls._ambariTezServiceConfigVersion) cls._modifiedConfigs.clear() cls._hiveServer2Port = None if len(services) > 0: cls.restartServices(services=services, waitForPortToOpen=waitForPortToOpen)
def stopService(cls, services=['metastore', 'hiveserver2']): for service in services: hosts = cls.getServiceHosts(service) for host in hosts: Ambari.stopComponent( host, cls._compSvcMap[service], waitForCompletion=True, timeout=cls._compSvcTimeouts[service] ) cls._postStopAction(service) # Workaround for BUG-73424 time.sleep(90)
def restartLLAP(cls): cluster = Ambari.getClusterName() data = '{"RequestInfo":{"context":"Restart LLAP","command":"RESTART_LLAP"},"Requests/resource_filters":[{"service_name":"HIVE","component_name":"HIVE_SERVER_INTERACTIVE","hosts":"%s"}]}' % ( cls.getHiveHost() ) url = '/api/v1/clusters/' + cluster + '/requests' r = Ambari.http_put_post_request(url, data, 'POST') assert (r.status_code == 200 or r.status_code == 201 or r.status_code == 202 ), "Failed to restart LLAP on host %s, status=%d" % (cls.getHiveHost(), r.status_code) time.sleep(240)
def start_stop_service(cls, service, state, cluster=None): if cluster == None: cluster = 'cl1' weburl = Ambari.getWebUrl() url = "%s/api/v1/clusters/%s/services/%s" % (weburl, cluster, service) auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword()) head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'} data = '{"RequestInfo": {"context": "DRUID API ' + state + ' ' + service + '"}, "Body" : {"ServiceInfo": {"state": "' + state + '"}}}' response = requests.put(url, data=data, headers=head, auth=auth, verify=False) assert ( response.status_code == 200 or response.status_code == 202 ),\ "Failed to start/stop service %s , status=%d" % ( service, response.status_code ) return response
def changePolicyInterval(cls): if not BeaconRanger.policyIntervalChanged: interval_for_source_cls = Ambari.getConfig( 'ranger-hive-security', webURL=source_weburl )['ranger.plugin.hive.policy.pollIntervalMs'] interval_for_target_cls = Ambari.getConfig( 'ranger-hive-security', webURL=target_weburl )['ranger.plugin.hive.policy.pollIntervalMs'] if not interval_for_source_cls == "5000": Xa.changePolicyInterval("HIVE", "5000", webURL=source_weburl) if not interval_for_target_cls == "5000": Xa.changePolicyInterval("HIVE", "5000", webURL=target_weburl) BeaconRanger.policyIntervalChanged = True BeaconRanger.policyActivationWaitTime = 6
def restart_all_services(cls, host, type, cluster=None): if cluster == None: cluster = 'cl1' weburl = cls.getweburl(host) url = "%s/api/v1/clusters/%s/services" % (weburl, cluster) auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword()) head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'} response = requests.get(url=url, headers=head, auth=auth, verify=False) json_data = json.loads(response._content) for i in range(0, len(json_data['items'])): service = json_data['items'][i]['ServiceInfo']['service_name'] logger.info("Restarting service : %s " % service) cls.restart_service(service, host)
def getDruidComponentHost(cls, component, service='DRUID', cluster=None): if cluster == None: cluster = 'cl1' host_names = [] url = "%s/api/v1/clusters/%s/services/%s/components/%s" % (Ambari.getWebUrl(), cluster, service, component) auth = HTTPBasicAuth(Ambari.getAdminUsername(), Ambari.getAdminPassword()) head = {'X-Requested-By': 'ambari', 'Content-Type': 'text/plain'} response = requests.get(url=url, headers=head, auth=auth, verify=False) assert response.status_code == 200, "Failed to get config versions." json = response.json() host_components = json['host_components'] for i in range(0, len(host_components)): host_names.append(host_components[i]['HostRoles']['host_name']) return host_names
def checkKafkaStatusOnMetronHost(cls, metron_host): kafka_status = None url = Ambari.getWebUrl( is_hdp=False ) + "/api/v1/clusters/%s/hosts/%s/host_components/%s" % ( cls.get_ambari_cluster_name(), metron_host, cls.kafka_broker) retcode, retdata, _retheaders = Ambari.performRESTCall(url) if retcode == 200: jsoncontent = util.getJSON(retdata) try: kafka_status = jsoncontent['HostRoles']['state'] except Exception: logger.error('Kafka broker is not running on the Metron host') kafka_status = None return kafka_status
def isSecuredCluster(cls): ''' Return whether the cluster is secured or not :return: boolean ''' cls._is_secure = Ambari.isSecure() return cls._is_secure
def getHiveLogDir(cls, logoutput=False): if cls._hiveLogDir is None: properties = Ambari.getConfig('hive-env') cls._hiveLogDir = properties['hive_log_dir'] if logoutput: logger.info("Hive.getHiveLogDir returns %s" % cls._hiveLogDir) return cls._hiveLogDir
def verify_Policy_Exists_after_replication( cls, servicetype, verify_from_cluster=source_weburl, custer_to_verify=target_weburl, database=None, path=None, NoPolicyInTarget=False, expectedDesc="created by beacon while importing from " + primaryCluster, preDenyPolicyStr=primaryCluster + "_beacon deny policy for "): if Xa.isArgusInstalled() is True: serviceName = "hadoop" if servicetype == "hdfs" else servicetype serviceNameOfverify_from_cluster = \ Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True, ambariWeburl=verify_from_cluster)[0]['name'] serviceNameOfverify_to_cluster = \ Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True, ambariWeburl=custer_to_verify)[0]['name'] logger.info("verifying if policy exist in target cluster") policies_in_source_Cluster = Xa.getPoliciesForResources( servicetype, serviceName=serviceNameOfverify_from_cluster, ambariWeburl=verify_from_cluster, database=database, path=path) policies_in_target_Cluster = Xa.getPoliciesForResources( servicetype, serviceName=serviceNameOfverify_to_cluster, ambariWeburl=custer_to_verify, database=database, path=path) if NoPolicyInTarget == False: assert len(policies_in_target_Cluster ) != 0, "make sure policies were imported" BeaconRanger.setIdOfAllPolicyToZero( policies_in_source_Cluster, policies_in_target_Cluster, expectedDesc) logger.info("set of policies in target cluster: " + str(policies_in_target_Cluster["policies"])) for policy in policies_in_source_Cluster["policies"]: logger.info("policy is " + str(policy)) assert policy in policies_in_target_Cluster["policies"] logger.info( "all policies are verified!! now will check for deny policy if it is true" ) isDenyPolicyTrue = Ambari.getConfig( 'beacon-security-site', webURL=source_weburl)['beacon.ranger.plugin.create.denypolicy'] all_policies_in_target_Cluster = Xa.getPolicy_api_v2( servicetype, weburl=target_weburl) if isDenyPolicyTrue == 'true': dataset = path if servicetype == "hdfs" else database BeaconRanger.denyPolicyValidation( servicetype, dataset, all_policies_in_target_Cluster, preDenyPolicyStr) else: assert len(policies_in_target_Cluster) == len( policies_in_source_Cluster)
def print_environment_details(self): UpgradeLogger.reportProgress("=====Environment Details=====\n ", True) is_hdp = self.STACK_TYPE.lower() == "hdp" UpgradeLogger.reportProgress( "Ambari URL : " + Ambari.getWebUrl(is_hdp=is_hdp), True) UpgradeLogger.reportProgress("Ambari OS : " + Ambari.getOS(), True) UpgradeLogger.reportProgress( "Stack Type : " + Config.get('ambari', 'STACK_TYPE'), True) UpgradeLogger.reportProgress( "Ambari DB : " + Config.get('ambari', 'AMBARI_DB'), True) UpgradeLogger.reportProgress( "Kerberos : " + Config.get('machine', 'IS_SECURE'), True) UpgradeLogger.reportProgress( "HA : " + Config.get('ambari', 'IS_HA_TEST'), True) UpgradeLogger.reportProgress( "Wire Encryption : " + Config.get('ambari', 'WIRE_ENCRYPTION'), True)
def getBaseUrl(self): from beaver.component.ambari import Ambari GRAFANA_HOST = Ambari.getHostsForComponent('METRICS_GRAFANA')[0] if Hadoop.isEncrypted() or Machine.isHumboldt(): GRAFANA_URL = "https://%s:3000/dashboard/db/hbase-tuning" % (GRAFANA_HOST) else: GRAFANA_URL = "http://%s:3000/dashboard/db/hbase-tuning" % (GRAFANA_HOST) self.base_url = GRAFANA_URL return self.base_url