def get_client_properties(self):
from beaver.component.kafka import Kafka
from beaver.component.ambari import Ambari
Kafka.alterUser(
userName=self.client_username,
config="'SCRAM-SHA-256=[iterations=8192,password=%s],"
"SCRAM-SHA-512=[password=%s]'" % (self.client_password, self.client_password)
)
is_ambari_enc = Ambari.is_ambari_encrypted()
kafka_jaas_config = Ambari.getConfig(
"kafka_jaas_conf", webURL=Ambari.getWebUrl(is_hdp=False, is_enc=is_ambari_enc)
)
replacement_jaas_entry = self.jaas_entry
if not Kafka._isSecure:
replacement_jaas_entry = "\nKafkaServer {%s\n};" % self.jaas_entry
if self.to_be_replaced + replacement_jaas_entry not in kafka_jaas_config['content']:
print "old : %s" % kafka_jaas_config['content']
kafka_jaas_config['content'] = kafka_jaas_config['content'].replace(
self.to_be_replaced, self.to_be_replaced + replacement_jaas_entry
)
print "new : %s" % kafka_jaas_config['content']
Ambari.setConfig(
"kafka_jaas_conf", kafka_jaas_config, webURL=Ambari.getWebUrl(is_hdp=False, is_enc=is_ambari_enc)
)
Ambari.restart_services_with_stale_configs()
time.sleep(20)
return {'sasl.jaas.config': self.jaas_entry.replace("\n", " "), 'sasl.mechanism': 'SCRAM-SHA-256'}
def fix_qe_14910(cls):
# QE-14190: replace ${hdp.version} in mapreduce.application.classpath because it will case bad substitution error
config = Ambari.getConfig(type='mapred-site')
for key in config.keys():
value = config[key]
value = value.replace('${hdp.version}', '{{version}}')
config[key] = value
Ambari.setConfig(type='mapred-site', config=config)
Ambari.restart_services_with_stale_configs()
def removeYarnQueue(cls,
queueName="newQueue",
defaultConfig=None,
webURL=source_weburl,
cluster=None):
if cluster == None:
cluster = Ambari.getClusterName()
if webURL == None:
webURL = Ambari.getWebUrl()
else:
cluster = Ambari.getClusterName(weburl=webURL)
# Delete new config
deleteConfig = {}
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".acl_administer_jobs"] = "*"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".acl_submit_applications"] = "*"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".capacity"] = "0"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".maximum-capacity"] = "0"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".state"] = "RUNNING"
deleteConfig["yarn.scheduler.capacity.root." + queueName +
".user-limit-factor"] = "1"
Ambari.deleteConfig(type='capacity-scheduler',
config=deleteConfig,
webURL=webURL)
# Reset to default config
Ambari.setConfig(type='capacity-scheduler',
config=defaultConfig,
webURL=webURL)
Ambari.start_stop_service('YARN',
'INSTALLED',
waitForCompletion=True,
weburl=webURL)
logger.info("---- Done stopping YARN cluster")
Ambari.start_stop_service('YARN',
'STARTED',
waitForCompletion=True,
weburl=webURL)
time.sleep(30)
logger.info("---- Done starting YARN cluster")
def modifyConfig(cls, changes, env={}, restartService=True):
logger.info("Current Service Config Version: %s" %
cls.getStartingAmbariServiceConfigVersion())
for key, value in changes.items():
if cls._ambariConfigMap.has_key(key):
key = cls._ambariConfigMap[key]
else:
logger.warn(
"Unknown config \"%s\" change requested, ignoring" % key)
continue
Ambari.setConfig(key, value)
if len(env.keys()) > 0:
key = cls._ambariConfigMap['oozie-env.sh']
envProps = Ambari.getConfig(key)
if envProps.has_key("content"):
content = envProps['content']
for envKey, envVal in env.items():
content = "export %s=%s\n%s" % (envKey, envVal, content)
Ambari.setConfig(key, {'content': content})
if restartService:
Ambari.restart_services_with_stale_configs()
def addYarnQueue(cls,
queueName="newQueue",
capacity=26,
webURL=source_weburl):
defaultYarnQueueConfig = Ambari.getConfig(service='YARN',
type='capacity-scheduler',
webURL=webURL)
updatedYarnQueueConfig = defaultYarnQueueConfig.copy()
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.queues"] = defaultYarnQueueConfig[
"yarn.scheduler.capacity.root.queues"] + "," + queueName
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.default.capacity"] = str(
int(defaultYarnQueueConfig[
"yarn.scheduler.capacity.root.default.capacity"].split(".")
[0]) - capacity)
updatedYarnQueueConfig[
"yarn.scheduler.capacity.root.default.maximum-capacity"] = str(
int(defaultYarnQueueConfig[
"yarn.scheduler.capacity.root.default.maximum-capacity"].
split(".")[0]) - capacity)
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".acl_administer_jobs"] = "*"
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".acl_submit_applications"] = "*"
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".capacity"] = str(capacity)
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".maximum-capacity"] = str(capacity)
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".state"] = "RUNNING"
updatedYarnQueueConfig["yarn.scheduler.capacity.root." + queueName +
".user-limit-factor"] = "1"
Ambari.setConfig(type='capacity-scheduler',
config=updatedYarnQueueConfig,
webURL=webURL)
logger.info("Add new YARN Queue: Successful")
return updatedYarnQueueConfig, defaultYarnQueueConfig
def disableEnableDenyPolicyCreation(cls, enableDenyPolicyCreation, webURL):
isDenyPolicyTrue = Ambari.getConfig(
'beacon-security-site',
webURL=webURL)['beacon.ranger.plugin.create.denypolicy']
if (isDenyPolicyTrue == 'true') == enableDenyPolicyCreation:
logger.info("skiping changing config , as it is already present")
else:
propsToSet = {
'beacon.ranger.plugin.create.denypolicy':
enableDenyPolicyCreation
}
Ambari.setConfig("beacon-security-site", propsToSet, webURL=webURL)
Ambari.start_stop_service("BEACON",
'INSTALLED',
waitForCompletion=True,
weburl=webURL)
logger.info("---- Done stopping Beacon cluster")
Ambari.start_stop_service('BEACON',
'STARTED',
waitForCompletion=True,
weburl=webURL)
logger.info("---- Done starting Beacon cluster")
def modifyConfig(
cls, changes, services=['metastore', 'hiveserver2'], env={}, restartService=True, waitForPortToOpen=True
):
logger.info("Current Service Config Version: %s" % cls.getStartingAmbariServiceConfigVersion())
for key, value in changes.items():
if cls._ambariConfigMap.has_key(key):
key = cls._ambariConfigMap[key]
else:
logger.warn("Unknown config \"%s\" change requested, ignoring" % key)
continue
Ambari.setConfig(key, value)
for k, v in value.items():
cls._modifiedConfigs[k] = v
if len(env.keys()) > 0:
key = cls._ambariConfigMap['hive-env.sh']
envProps = Ambari.getConfig(key)
if envProps.has_key("content"):
content = envProps['content']
for envKey, envVal in env.items():
content = "export %s=%s\n%s" % (envKey, envVal, content)
Ambari.setConfig(key, {'content': content})
if restartService:
cls.restartServices(services=services, waitForPortToOpen=waitForPortToOpen)