def changePolicyInterval(cls):
if not BeaconRanger.policyIntervalChanged:
interval_for_source_cls = Ambari.getConfig(
'ranger-hive-security', webURL=source_weburl
)['ranger.plugin.hive.policy.pollIntervalMs']
interval_for_target_cls = Ambari.getConfig(
'ranger-hive-security', webURL=target_weburl
)['ranger.plugin.hive.policy.pollIntervalMs']
if not interval_for_source_cls == "5000":
Xa.changePolicyInterval("HIVE", "5000", webURL=source_weburl)
if not interval_for_target_cls == "5000":
Xa.changePolicyInterval("HIVE", "5000", webURL=target_weburl)
BeaconRanger.policyIntervalChanged = True
BeaconRanger.policyActivationWaitTime = 6
def get_client_properties(self):
from beaver.component.kafka import Kafka
from beaver.component.ambari import Ambari
Kafka.alterUser(
userName=self.client_username,
config="'SCRAM-SHA-256=[iterations=8192,password=%s],"
"SCRAM-SHA-512=[password=%s]'" % (self.client_password, self.client_password)
)
is_ambari_enc = Ambari.is_ambari_encrypted()
kafka_jaas_config = Ambari.getConfig(
"kafka_jaas_conf", webURL=Ambari.getWebUrl(is_hdp=False, is_enc=is_ambari_enc)
)
replacement_jaas_entry = self.jaas_entry
if not Kafka._isSecure:
replacement_jaas_entry = "\nKafkaServer {%s\n};" % self.jaas_entry
if self.to_be_replaced + replacement_jaas_entry not in kafka_jaas_config['content']:
print "old : %s" % kafka_jaas_config['content']
kafka_jaas_config['content'] = kafka_jaas_config['content'].replace(
self.to_be_replaced, self.to_be_replaced + replacement_jaas_entry
)
print "new : %s" % kafka_jaas_config['content']
Ambari.setConfig(
"kafka_jaas_conf", kafka_jaas_config, webURL=Ambari.getWebUrl(is_hdp=False, is_enc=is_ambari_enc)
)
Ambari.restart_services_with_stale_configs()
time.sleep(20)
return {'sasl.jaas.config': self.jaas_entry.replace("\n", " "), 'sasl.mechanism': 'SCRAM-SHA-256'}
def getHiveLogDir(cls, logoutput=False):
if cls._hiveLogDir is None:
properties = Ambari.getConfig('hive-env')
cls._hiveLogDir = properties['hive_log_dir']
if logoutput:
logger.info("Hive.getHiveLogDir returns %s" % cls._hiveLogDir)
return cls._hiveLogDir
def verify_Policy_Exists_after_replication(
cls,
servicetype,
verify_from_cluster=source_weburl,
custer_to_verify=target_weburl,
database=None,
path=None,
NoPolicyInTarget=False,
expectedDesc="created by beacon while importing from " +
primaryCluster,
preDenyPolicyStr=primaryCluster + "_beacon deny policy for "):
if Xa.isArgusInstalled() is True:
serviceName = "hadoop" if servicetype == "hdfs" else servicetype
serviceNameOfverify_from_cluster = \
Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True,
ambariWeburl=verify_from_cluster)[0]['name']
serviceNameOfverify_to_cluster = \
Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True,
ambariWeburl=custer_to_verify)[0]['name']
logger.info("verifying if policy exist in target cluster")
policies_in_source_Cluster = Xa.getPoliciesForResources(
servicetype,
serviceName=serviceNameOfverify_from_cluster,
ambariWeburl=verify_from_cluster,
database=database,
path=path)
policies_in_target_Cluster = Xa.getPoliciesForResources(
servicetype,
serviceName=serviceNameOfverify_to_cluster,
ambariWeburl=custer_to_verify,
database=database,
path=path)
if NoPolicyInTarget == False:
assert len(policies_in_target_Cluster
) != 0, "make sure policies were imported"
BeaconRanger.setIdOfAllPolicyToZero(
policies_in_source_Cluster, policies_in_target_Cluster,
expectedDesc)
logger.info("set of policies in target cluster: " +
str(policies_in_target_Cluster["policies"]))
for policy in policies_in_source_Cluster["policies"]:
logger.info("policy is " + str(policy))
assert policy in policies_in_target_Cluster["policies"]
logger.info(
"all policies are verified!! now will check for deny policy if it is true"
)
isDenyPolicyTrue = Ambari.getConfig(
'beacon-security-site',
webURL=source_weburl)['beacon.ranger.plugin.create.denypolicy']
all_policies_in_target_Cluster = Xa.getPolicy_api_v2(
servicetype, weburl=target_weburl)
if isDenyPolicyTrue == 'true':
dataset = path if servicetype == "hdfs" else database
BeaconRanger.denyPolicyValidation(
servicetype, dataset, all_policies_in_target_Cluster,
preDenyPolicyStr)
else:
assert len(policies_in_target_Cluster) == len(
policies_in_source_Cluster)
def fix_qe_14910(cls):
# QE-14190: replace ${hdp.version} in mapreduce.application.classpath because it will case bad substitution error
config = Ambari.getConfig(type='mapred-site')
for key in config.keys():
value = config[key]
value = value.replace('${hdp.version}', '{{version}}')
config[key] = value
Ambari.setConfig(type='mapred-site', config=config)
Ambari.restart_services_with_stale_configs()
def get_port(cls):
"""
Returns the port on which smm is running
"""
if not cls._rest_port:
conf_from_ambari = Ambari.getConfig(service='STREAMSMSGMGR', type="streams-messaging-manager-common")
cls._rest_port = conf_from_ambari['adminPort']
logger.info("Server port is = %s" % cls._rest_port)
cls._rest_port = int(cls._rest_port)
return cls._rest_port
def get_port(cls):
"""
Returns the port on which streamline is running
"""
if not cls._rest_port:
conf_from_ambari = Ambari.getConfig(service='STREAMLINE', type="streamline-common")
cls._rest_port = conf_from_ambari['port']
logger.info("streamline server port is = %s" % cls._rest_port)
cls._rest_port = int(cls._rest_port)
return cls._rest_port
def get_port(cls):
"""
Returns the port on which schema registry is running
"""
if not cls._rest_port:
conf_from_ambari = Ambari.getConfig(service='REGISTRY',
type="registry-common")
cls._rest_port = conf_from_ambari['port']
logger.info("Schema registry server port is = %s", cls._rest_port)
cls._rest_port = int(cls._rest_port)
return cls._rest_port
def Atlas_get_log_dir(cls, logoutput=True):
try:
# TODO: change it to read from atlas-env.sh
from beaver.component.ambari import Ambari
return Ambari.getConfig("atlas-env")['metadata_log_dir']
except Exception as e:
if logoutput:
logger.error(
"Exception occured during Atlas_get_log_dir() call: %s",
str(e))
return None
def setup_for_ranger():
global ranger_orig_url, admin_prop_loc
if (isRangerInstalled()):
CONF['XA_ADMIN_HOST'] = Ambari.getConfig(
'admin-properties',
webURL=ambari_url)['policymgr_external_url'].split('//',
1)[1].split(
':', 1)[0]
if Hadoop.isEncrypted():
CONF['XA_ADMIN_PORT'] = Ambari.getConfig(
'ranger-admin-site',
webURL=ambari_url)['ranger.service.https.port']
else:
CONF['XA_ADMIN_PORT'] = Ambari.getConfig(
'ranger-admin-site',
webURL=ambari_url)['ranger.service.http.port']
admin_prop_loc = os.path.join(_workspace, _artifact,
"xaagents-knoxsso", "knox-sso-ui",
"Test.properties")
ranger_orig_url = "HOST=http://" + CONF['XA_ADMIN_HOST'] + ":" + str(
CONF['XA_ADMIN_PORT']) + "' " + admin_prop_loc
def getYarnQueue(cls, job_id=None, webURL=source_weburl, beaconURL=None):
yarnSiteConfig = Ambari.getConfig(service='YARN',
type='yarn-site',
webURL=webURL)
yarnHostPort = yarnSiteConfig["yarn.resourcemanager.webapp.address"]
yarnHttpPolicy = yarnSiteConfig["yarn.http.policy"]
protocol = None
if yarnHttpPolicy == "HTTP_ONLY":
protocol = "http"
elif yarnHttpPolicy == "HTTPS_ONLY":
protocol = "https"
yarn_webURL = protocol + "://" + yarnHostPort
uri = "/ws/v1/cluster/apps/application_" + job_id
logger.info("Get Yarn application details: " + yarn_webURL + uri)
response = Beacon.doGetWithBeaconSession(url=yarn_webURL + uri,
beacon_url=beaconURL)
logger.info("Get Yarn application details response: " +
response._content)
response_json = response.json()
return response_json["app"]["queue"]
def getDatabaseFlavor(cls):
properties = Ambari.getConfig('druid-common', service='DRUID')
connectURI = properties['druid.metadata.storage.connector.connectURI']
if connectURI.find("oracle") != -1:
return "oracle"
elif connectURI.find("postgresql") != -1:
m = re.search("jdbc:postgresql://(.*):.*", connectURI)
dbHost = Machine.getfqdn()
if m and m.group(1):
dbHost = m.group(1)
dbVersion = Machine.getDBVersion('postgres', host=dbHost)
if dbVersion:
return "postgres-%s" % dbVersion
else:
return "postgres"
elif connectURI.find("derby") != -1:
return "derby"
elif connectURI.find("mysql") != -1:
return "mysql"
return ""
def modifyConfig(cls, changes, env={}, restartService=True):
logger.info("Current Service Config Version: %s" %
cls.getStartingAmbariServiceConfigVersion())
for key, value in changes.items():
if cls._ambariConfigMap.has_key(key):
key = cls._ambariConfigMap[key]
else:
logger.warn(
"Unknown config \"%s\" change requested, ignoring" % key)
continue
Ambari.setConfig(key, value)
if len(env.keys()) > 0:
key = cls._ambariConfigMap['oozie-env.sh']
envProps = Ambari.getConfig(key)
if envProps.has_key("content"):
content = envProps['content']
for envKey, envVal in env.items():
content = "export %s=%s\n%s" % (envKey, envVal, content)
Ambari.setConfig(key, {'content': content})
if restartService:
Ambari.restart_services_with_stale_configs()
def addYarnQueue(cls,
queueName="newQueue",
capacity=26,
webURL=source_weburl):
defaultYarnQueueConfig = Ambari.getConfig(service='YARN',
type='capacity-scheduler',
webURL=webURL)
updatedYarnQueueConfig = defaultYarnQueueConfig.copy()
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.queues"] = defaultYarnQueueConfig[
"yarn.scheduler.capacity.root.queues"] + "," + queueName
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.default.capacity"] = str(
int(defaultYarnQueueConfig[
"yarn.scheduler.capacity.root.default.capacity"].split(".")
[0]) - capacity)
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.default.maximum-capacity"] = str(
int(defaultYarnQueueConfig[
"yarn.scheduler.capacity.root.default.maximum-capacity"].
split(".")[0]) - capacity)
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".acl_administer_jobs"] = "*"
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".acl_submit_applications"] = "*"
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".capacity"] = str(capacity)
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".maximum-capacity"] = str(capacity)
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".state"] = "RUNNING"
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".user-limit-factor"] = "1"
Ambari.setConfig(type='capacity-scheduler',
config=updatedYarnQueueConfig,
webURL=webURL)
logger.info("Add new YARN Queue: Successful")
return updatedYarnQueueConfig, defaultYarnQueueConfig
def disableEnableDenyPolicyCreation(cls, enableDenyPolicyCreation, webURL):
isDenyPolicyTrue = Ambari.getConfig(
'beacon-security-site',
webURL=webURL)['beacon.ranger.plugin.create.denypolicy']
if (isDenyPolicyTrue == 'true') == enableDenyPolicyCreation:
logger.info("skiping changing config , as it is already present")
else:
propsToSet = {
'beacon.ranger.plugin.create.denypolicy':
enableDenyPolicyCreation
}
Ambari.setConfig("beacon-security-site", propsToSet, webURL=webURL)
Ambari.start_stop_service("BEACON",
'INSTALLED',
waitForCompletion=True,
weburl=webURL)
logger.info("---- Done stopping Beacon cluster")
Ambari.start_stop_service('BEACON',
'STARTED',
waitForCompletion=True,
weburl=webURL)
logger.info("---- Done starting Beacon cluster")
def modifyConfig(
cls, changes, services=['metastore', 'hiveserver2'], env={}, restartService=True, waitForPortToOpen=True
):
logger.info("Current Service Config Version: %s" % cls.getStartingAmbariServiceConfigVersion())
for key, value in changes.items():
if cls._ambariConfigMap.has_key(key):
key = cls._ambariConfigMap[key]
else:
logger.warn("Unknown config \"%s\" change requested, ignoring" % key)
continue
Ambari.setConfig(key, value)
for k, v in value.items():
cls._modifiedConfigs[k] = v
if len(env.keys()) > 0:
key = cls._ambariConfigMap['hive-env.sh']
envProps = Ambari.getConfig(key)
if envProps.has_key("content"):
content = envProps['content']
for envKey, envVal in env.items():
content = "export %s=%s\n%s" % (envKey, envVal, content)
Ambari.setConfig(key, {'content': content})
if restartService:
cls.restartServices(services=services, waitForPortToOpen=waitForPortToOpen)
def getConfigValueViaAmbari(cls, type, property_name, cluster=None):
type_config = Ambari.getConfig(type, service='HIVE')
param_vale = type_config[property_name]
return param_vale
def getZKPort(cls):
if not cls._zkPort:
zoo_props = Ambari.getConfig("zoo.cfg")
if zoo_props:
cls._zkPort = zoo_props["clientPort"]
return cls._zkPort