def test_convert_yaml_array(self): expected_values = [] expected_values.append("c6401.ambari.apache.org") values = yaml_utils.get_values_from_yaml_array("['c6401.ambari.apache.org']") self.assertEquals(expected_values, values) expected_values.append("c6402.ambari.apache.org") values = yaml_utils.get_values_from_yaml_array("['c6401.ambari.apache.org', 'c6402.ambari.apache.org']") self.assertEquals(expected_values, values) values = yaml_utils.get_values_from_yaml_array('["c6401.ambari.apache.org", "c6402.ambari.apache.org"]') self.assertEquals(expected_values, values) values = yaml_utils.get_values_from_yaml_array('[\'c6401.ambari.apache.org\', "c6402.ambari.apache.org"]') self.assertEquals(expected_values, values)
def delete_storm_zookeeper_data(self, env): """ Deletes the Storm data from ZooKeeper, effectively bringing down all Storm daemons. :return: """ import params Logger.info('Clearing Storm data from ZooKeeper') storm_zookeeper_root_dir = params.storm_zookeeper_root_dir if storm_zookeeper_root_dir is None: raise Fail("The storm ZooKeeper directory specified by storm-site/storm.zookeeper.root must be specified") # the zookeeper client must be given a zookeeper host to contact storm_zookeeper_server_list = yaml_utils.get_values_from_yaml_array(params.storm_zookeeper_servers) if storm_zookeeper_server_list is None: Logger.info("Unable to extract ZooKeeper hosts from '{0}', assuming localhost").format(params.storm_zookeeper_servers) storm_zookeeper_server_list = ["localhost"] # for every zk server, try to remove /storm zookeeper_data_cleared = False for storm_zookeeper_server in storm_zookeeper_server_list: # determine where the zkCli.sh shell script is zk_command_location = "/usr/hdp/current/zookeeper-client/bin/zkCli.sh" if params.version is not None: zk_command_location = "/usr/hdp/{0}/zookeeper/bin/zkCli.sh".format(params.version) # create the ZooKeeper delete command command = "{0} -server {1}:{2} rmr /storm".format( zk_command_location, storm_zookeeper_server, params.storm_zookeeper_port) # clean out ZK try: # the ZK client requires Java to run; ensure it's on the path env_map = { 'JAVA_HOME': params.java64_home } # AMBARI-12094: if security is enabled, then we need to tell zookeeper where the # JAAS file is located since we don't use kinit directly with STORM if params.security_enabled: env_map['JVMFLAGS'] = "-Djava.security.auth.login.config={0}".format(params.storm_jaas_file) Execute(command, user=params.storm_user, environment=env_map, logoutput=True, tries=1) zookeeper_data_cleared = True break except: # the command failed, try a different ZK server pass # fail if the ZK data could not be cleared if not zookeeper_data_cleared: raise Fail("Unable to clear ZooKeeper Storm data on any of the following ZooKeeper hosts: {0}".format( storm_zookeeper_server_list))
def delete_storm_zookeeper_data(self, env): """ Deletes the Storm data from ZooKeeper, effectively bringing down all Storm daemons. :return: """ import params Logger.info('Clearing Storm data from ZooKeeper') storm_zookeeper_root_dir = params.storm_zookeeper_root_dir if storm_zookeeper_root_dir is None: raise Fail("The storm ZooKeeper directory specified by storm-site/storm.zookeeper.root must be specified") # The zookeeper client must be given a zookeeper host to contact. Guaranteed to have at least one host. storm_zookeeper_server_list = yaml_utils.get_values_from_yaml_array(params.storm_zookeeper_servers) if storm_zookeeper_server_list is None: Logger.info("Unable to extract ZooKeeper hosts from '{0}', assuming localhost").format(params.storm_zookeeper_servers) storm_zookeeper_server_list = ["localhost"] # For every zk server, try to remove /storm zookeeper_data_cleared = False for storm_zookeeper_server in storm_zookeeper_server_list: # Determine where the zkCli.sh shell script is zk_command_location = "/usr/hdp/current/zookeeper-client/bin/zkCli.sh" if params.version is not None: zk_command_location = "/usr/hdp/{0}/zookeeper/bin/zkCli.sh".format(params.version) # create the ZooKeeper delete command command = "{0} -server {1}:{2} rmr /storm".format( zk_command_location, storm_zookeeper_server, params.storm_zookeeper_port) # clean out ZK try: # the ZK client requires Java to run; ensure it's on the path env_map = { 'JAVA_HOME': params.java64_home } # AMBARI-12094: if security is enabled, then we need to tell zookeeper where the # JAAS file is located since we don't use kinit directly with STORM if params.security_enabled: env_map['JVMFLAGS'] = "-Djava.security.auth.login.config={0}".format(params.storm_jaas_file) Execute(command, user=params.storm_user, environment=env_map, logoutput=True, tries=1) zookeeper_data_cleared = True break except: # the command failed, try a different ZK server pass # fail if the ZK data could not be cleared if not zookeeper_data_cleared: raise Fail("Unable to clear ZooKeeper Storm data on any of the following ZooKeeper hosts: {0}".format( storm_zookeeper_server_list))
def test_convert_yaml_array(self): expected_values = [] expected_values.append("c6401.ambari.apache.org") values = yaml_utils.get_values_from_yaml_array( "['c6401.ambari.apache.org']") self.assertEquals(expected_values, values) expected_values.append("c6402.ambari.apache.org") values = yaml_utils.get_values_from_yaml_array( "['c6401.ambari.apache.org', 'c6402.ambari.apache.org']") self.assertEquals(expected_values, values) values = yaml_utils.get_values_from_yaml_array( '["c6401.ambari.apache.org", "c6402.ambari.apache.org"]') self.assertEquals(expected_values, values) values = yaml_utils.get_values_from_yaml_array( '[\'c6401.ambari.apache.org\', "c6402.ambari.apache.org"]') self.assertEquals(expected_values, values)
host_in_memory_aggregation_port = default( "/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) is_aggregation_https_enabled = False if default( "/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": host_in_memory_aggregation_protocol = 'https' is_aggregation_https_enabled = True else: host_in_memory_aggregation_protocol = 'http' # Cluster Zookeeper quorum zookeeper_quorum = "" if storm_zookeeper_servers: storm_zookeeper_servers_list = yaml_utils.get_values_from_yaml_array( storm_zookeeper_servers) zookeeper_quorum = (":" + storm_zookeeper_port + ",").join(storm_zookeeper_servers_list) zookeeper_quorum += ":" + storm_zookeeper_port jar_jvm_opts = '' ######################################################## ############# Atlas related params ##################### ######################################################## #region Atlas Hooks storm_atlas_application_properties = default( '/configurations/storm-atlas-application.properties', {}) enable_atlas_hook = default('/configurations/storm-env/storm.atlas.hook', False) atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file',