def test_convert_yaml_array(self):
expected_values = []
expected_values.append("c6401.ambari.apache.org")
values = yaml_utils.get_values_from_yaml_array("['c6401.ambari.apache.org']")
self.assertEquals(expected_values, values)
expected_values.append("c6402.ambari.apache.org")
values = yaml_utils.get_values_from_yaml_array("['c6401.ambari.apache.org', 'c6402.ambari.apache.org']")
self.assertEquals(expected_values, values)
values = yaml_utils.get_values_from_yaml_array('["c6401.ambari.apache.org", "c6402.ambari.apache.org"]')
self.assertEquals(expected_values, values)
values = yaml_utils.get_values_from_yaml_array('[\'c6401.ambari.apache.org\', "c6402.ambari.apache.org"]')
self.assertEquals(expected_values, values)
def delete_storm_zookeeper_data(self, env):
"""
Deletes the Storm data from ZooKeeper, effectively bringing down all
Storm daemons.
:return:
"""
import params
Logger.info('Clearing Storm data from ZooKeeper')
storm_zookeeper_root_dir = params.storm_zookeeper_root_dir
if storm_zookeeper_root_dir is None:
raise Fail("The storm ZooKeeper directory specified by storm-site/storm.zookeeper.root must be specified")
# the zookeeper client must be given a zookeeper host to contact
storm_zookeeper_server_list = yaml_utils.get_values_from_yaml_array(params.storm_zookeeper_servers)
if storm_zookeeper_server_list is None:
Logger.info("Unable to extract ZooKeeper hosts from '{0}', assuming localhost").format(params.storm_zookeeper_servers)
storm_zookeeper_server_list = ["localhost"]
# for every zk server, try to remove /storm
zookeeper_data_cleared = False
for storm_zookeeper_server in storm_zookeeper_server_list:
# determine where the zkCli.sh shell script is
zk_command_location = "/usr/hdp/current/zookeeper-client/bin/zkCli.sh"
if params.version is not None:
zk_command_location = "/usr/hdp/{0}/zookeeper/bin/zkCli.sh".format(params.version)
# create the ZooKeeper delete command
command = "{0} -server {1}:{2} rmr /storm".format(
zk_command_location, storm_zookeeper_server, params.storm_zookeeper_port)
# clean out ZK
try:
# the ZK client requires Java to run; ensure it's on the path
env_map = {
'JAVA_HOME': params.java64_home
}
# AMBARI-12094: if security is enabled, then we need to tell zookeeper where the
# JAAS file is located since we don't use kinit directly with STORM
if params.security_enabled:
env_map['JVMFLAGS'] = "-Djava.security.auth.login.config={0}".format(params.storm_jaas_file)
Execute(command, user=params.storm_user, environment=env_map,
logoutput=True, tries=1)
zookeeper_data_cleared = True
break
except:
# the command failed, try a different ZK server
pass
# fail if the ZK data could not be cleared
if not zookeeper_data_cleared:
raise Fail("Unable to clear ZooKeeper Storm data on any of the following ZooKeeper hosts: {0}".format(
storm_zookeeper_server_list))
def delete_storm_zookeeper_data(self, env):
"""
Deletes the Storm data from ZooKeeper, effectively bringing down all
Storm daemons.
:return:
"""
import params
Logger.info('Clearing Storm data from ZooKeeper')
storm_zookeeper_root_dir = params.storm_zookeeper_root_dir
if storm_zookeeper_root_dir is None:
raise Fail("The storm ZooKeeper directory specified by storm-site/storm.zookeeper.root must be specified")
# The zookeeper client must be given a zookeeper host to contact. Guaranteed to have at least one host.
storm_zookeeper_server_list = yaml_utils.get_values_from_yaml_array(params.storm_zookeeper_servers)
if storm_zookeeper_server_list is None:
Logger.info("Unable to extract ZooKeeper hosts from '{0}', assuming localhost").format(params.storm_zookeeper_servers)
storm_zookeeper_server_list = ["localhost"]
# For every zk server, try to remove /storm
zookeeper_data_cleared = False
for storm_zookeeper_server in storm_zookeeper_server_list:
# Determine where the zkCli.sh shell script is
zk_command_location = "/usr/hdp/current/zookeeper-client/bin/zkCli.sh"
if params.version is not None:
zk_command_location = "/usr/hdp/{0}/zookeeper/bin/zkCli.sh".format(params.version)
# create the ZooKeeper delete command
command = "{0} -server {1}:{2} rmr /storm".format(
zk_command_location, storm_zookeeper_server, params.storm_zookeeper_port)
# clean out ZK
try:
# the ZK client requires Java to run; ensure it's on the path
env_map = {
'JAVA_HOME': params.java64_home
}
# AMBARI-12094: if security is enabled, then we need to tell zookeeper where the
# JAAS file is located since we don't use kinit directly with STORM
if params.security_enabled:
env_map['JVMFLAGS'] = "-Djava.security.auth.login.config={0}".format(params.storm_jaas_file)
Execute(command, user=params.storm_user, environment=env_map,
logoutput=True, tries=1)
zookeeper_data_cleared = True
break
except:
# the command failed, try a different ZK server
pass
# fail if the ZK data could not be cleared
if not zookeeper_data_cleared:
raise Fail("Unable to clear ZooKeeper Storm data on any of the following ZooKeeper hosts: {0}".format(
storm_zookeeper_server_list))
def test_convert_yaml_array(self):
expected_values = []
expected_values.append("c6401.ambari.apache.org")
values = yaml_utils.get_values_from_yaml_array(
"['c6401.ambari.apache.org']")
self.assertEquals(expected_values, values)
expected_values.append("c6402.ambari.apache.org")
values = yaml_utils.get_values_from_yaml_array(
"['c6401.ambari.apache.org', 'c6402.ambari.apache.org']")
self.assertEquals(expected_values, values)
values = yaml_utils.get_values_from_yaml_array(
'["c6401.ambari.apache.org", "c6402.ambari.apache.org"]')
self.assertEquals(expected_values, values)
values = yaml_utils.get_values_from_yaml_array(
'[\'c6401.ambari.apache.org\', "c6402.ambari.apache.org"]')
self.assertEquals(expected_values, values)
host_in_memory_aggregation_port = default(
"/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port",
61888)
is_aggregation_https_enabled = False
if default(
"/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy",
"HTTP_ONLY") == "HTTPS_ONLY":
host_in_memory_aggregation_protocol = 'https'
is_aggregation_https_enabled = True
else:
host_in_memory_aggregation_protocol = 'http'
# Cluster Zookeeper quorum
zookeeper_quorum = ""
if storm_zookeeper_servers:
storm_zookeeper_servers_list = yaml_utils.get_values_from_yaml_array(
storm_zookeeper_servers)
zookeeper_quorum = (":" + storm_zookeeper_port +
",").join(storm_zookeeper_servers_list)
zookeeper_quorum += ":" + storm_zookeeper_port
jar_jvm_opts = ''
########################################################
############# Atlas related params #####################
########################################################
#region Atlas Hooks
storm_atlas_application_properties = default(
'/configurations/storm-atlas-application.properties', {})
enable_atlas_hook = default('/configurations/storm-env/storm.atlas.hook',
False)
atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file',
