def _load_jmx(self, ssl, host, port, jmx_metric): """ creates a JmxMetric object that holds info about jmx-based metrics """ value_list = [] kerberos_keytab = None kerberos_principal = None if logger.isEnabledFor(logging.DEBUG): logger.debug(str(jmx_metric.property_map)) security_enabled = str(self._get_configuration_value(SECURITY_ENABLED_KEY)).upper() == 'TRUE' if self.uri_property_keys.kerberos_principal is not None: kerberos_principal = self._get_configuration_value( self.uri_property_keys.kerberos_principal) if kerberos_principal is not None: # substitute _HOST in kerberos principal with actual fqdn kerberos_principal = kerberos_principal.replace('_HOST', self.host_name) if self.uri_property_keys.kerberos_keytab is not None: kerberos_keytab = self._get_configuration_value(self.uri_property_keys.kerberos_keytab) if "0.0.0.0" in str(host): host = self.host_name for jmx_property_key, jmx_property_value in jmx_metric.property_map.iteritems(): url = "{0}://{1}:{2}/jmx?qry={3}".format( "https" if ssl else "http", host, str(port), jmx_property_key) # use a customer header processor that will look for the non-standard # "Refresh" header and attempt to follow the redirect response = None content = '' try: if kerberos_principal is not None and kerberos_keytab is not None and security_enabled: tmp_dir = Constants.AGENT_TMP_DIR if tmp_dir is None: tmp_dir = gettempdir() kerberos_executable_search_paths = self._get_configuration_value('{{kerberos-env/executable_search_paths}}') smokeuser = self._get_configuration_value('{{cluster-env/smokeuser}}') response, error_msg, time_millis = curl_krb_request(tmp_dir, kerberos_keytab, kerberos_principal, url, "metric_alert", kerberos_executable_search_paths, False, self.get_name(), smokeuser, connection_timeout=self.curl_connection_timeout, kinit_timer_ms = self.kinit_timeout) content = response else: url_opener = urllib2.build_opener(RefreshHeaderProcessor()) response = url_opener.open(url, timeout=self.connection_timeout) content = response.read() except Exception, exception: if logger.isEnabledFor(logging.DEBUG): logger.exception("[Alert][{0}] Unable to make a web request: {1}".format(self.get_name(), str(exception))) finally:
def test_metric_alert_uses_refresh_processor(self, http_response_mock, http_connection_mock): """ Tests that the RefreshHeaderProcessor is correctly chained and called :param http_response_mock: :param http_connection_mock: :return: """ http_conn = http_connection_mock.return_value http_conn.getresponse.return_value = MagicMock(status=200) http_response_mock.return_value = MagicMock(code=200) url_opener = urllib2.build_opener(RefreshHeaderProcessor()) response = url_opener.open("http://foo.bar.baz/jmx") self.assertFalse(response is None) self.assertTrue(http_conn.request.called) self.assertTrue(http_conn.getresponse.called) self.assertTrue(http_response_mock.called) # now we know that the refresh header is intercepting, reset the mocks # and try with a METRIC alert MagicMock.reset_mock(http_response_mock) MagicMock.reset_mock(http_connection_mock) definition_json = self._get_metric_alert_definition() configuration = { 'hdfs-site': { 'dfs.datanode.http.address': 'c6401.ambari.apache.org:80' } } collector = AlertCollector() cluster_configuration = self.__get_cluster_configuration() self.__update_cluster_configuration(cluster_configuration, configuration) alert = MetricAlert(definition_json, definition_json['source']) alert.set_helpers(collector, cluster_configuration) alert.set_cluster("c1", "c6401.ambari.apache.org") alert.collect() self.assertFalse(response is None) self.assertTrue(http_conn.request.called) self.assertTrue(http_conn.getresponse.called) self.assertTrue(http_response_mock.called)
def get_value_from_jmx(query, jmx_property, connection_timeout): response = None try: # use a customer header process that will look for the non-standard # "Refresh" header and attempt to follow the redirect url_opener = urllib2.build_opener(RefreshHeaderProcessor()) response = url_opener.open(query, timeout=connection_timeout) data = response.read() data_dict = json.loads(data) return data_dict["beans"][0][jmx_property] finally: if response is not None: try: response.close() except: pass
def _load_jmx(self, ssl, host, port, jmx_metric): """ creates a JmxMetric object that holds info about jmx-based metrics """ value_list = [] if logger.isEnabledFor(logging.DEBUG): logger.debug(str(jmx_metric.property_map)) for jmx_property_key, jmx_property_value in jmx_metric.property_map.iteritems( ): url = "{0}://{1}:{2}/jmx?qry={3}".format( "https" if ssl else "http", host, str(port), jmx_property_key) # use a customer header processor that will look for the non-standard # "Refresh" header and attempt to follow the redirect response = None try: url_opener = urllib2.build_opener(RefreshHeaderProcessor()) response = url_opener.open(url, timeout=CONNECTION_TIMEOUT) content = response.read() finally: # explicitely close the connection as we've seen python hold onto these if response is not None: try: response.close() except: logger.debug( "[Alert][{0}] Unable to close JMX URL connection to {1}" .format(self.get_name(), url)) json_response = json.loads(content) json_data = json_response['beans'][0] for attr in jmx_property_value: if attr not in json_data: raise Exception( "Unable to find {0} in JSON from {1} ".format( attr, url)) value_list.append(json_data[attr]) return value_list
def test_urllib2_refresh_header_processor(self): from urllib2 import Request # setup the original request original_url = "http://foo.bar.baz/jmx?qry=someQuery" request = Request(original_url) # ensure that we get back a 200 with a refresh header to redirect us response = MagicMock(code=200) info_response = MagicMock() info_response.keys.return_value = ["Refresh"] info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080" response.info.return_value = info_response # add a mock parent to the refresh processor parent_mock = MagicMock() refresh_processor = RefreshHeaderProcessor() refresh_processor.parent = parent_mock # execute refresh_processor.http_response(request, response) # ensure that the parent was called with the modified URL parent_mock.open.assert_called_with( "http://foobar.baz.qux:8080/jmx?qry=someQuery") # reset mocks MagicMock.reset_mock(parent_mock) # alter the refresh header to remove the time value info_response.getheader.return_value = "url=http://foobar.baz.qux:8443" # execute refresh_processor.http_response(request, response) # ensure that the parent was called with the modified URL parent_mock.open.assert_called_with( "http://foobar.baz.qux:8443/jmx?qry=someQuery") # reset mocks MagicMock.reset_mock(parent_mock) # use an invalid refresh header info_response.getheader.return_value = "http://foobar.baz.qux:8443" # execute refresh_processor.http_response(request, response) # ensure that the parent was not called self.assertFalse(parent_mock.open.called) # reset mocks MagicMock.reset_mock(parent_mock) # remove the refresh header info_response.keys.return_value = ["SomeOtherHeaders"] # execute refresh_processor.http_response(request, response) # ensure that the parent was not called self.assertFalse(parent_mock.open.called) # reset mocks MagicMock.reset_mock(parent_mock) # use and invalid http code but include a refresh header response.code = 401 info_response.keys.return_value = ["Refresh"] info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080" # execute refresh_processor.http_response(request, response) # ensure that the parent was not called self.assertFalse(parent_mock.open.called)
def test_urllib2_refresh_header_processor(self): from urllib2 import Request # setup the original request original_url = "http://foo.bar.baz/jmx?qry=someQuery" request = Request(original_url) # ensure that we get back a 200 with a refresh header to redirect us response = MagicMock(code=200) info_response = MagicMock() info_response.keys.return_value = ["Refresh"] info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080" response.info.return_value = info_response # add a mock parent to the refresh processor parent_mock = MagicMock() refresh_processor = RefreshHeaderProcessor() refresh_processor.parent = parent_mock # execute refresh_processor.http_response(request, response) # ensure that the parent was called with the modified URL parent_mock.open.assert_called_with("http://foobar.baz.qux:8080/jmx?qry=someQuery") # reset mocks MagicMock.reset_mock(parent_mock) # alter the refresh header to remove the time value info_response.getheader.return_value = "url=http://foobar.baz.qux:8443" # execute refresh_processor.http_response(request, response) # ensure that the parent was called with the modified URL parent_mock.open.assert_called_with("http://foobar.baz.qux:8443/jmx?qry=someQuery") # reset mocks MagicMock.reset_mock(parent_mock) # use an invalid refresh header info_response.getheader.return_value = "http://foobar.baz.qux:8443" # execute refresh_processor.http_response(request, response) # ensure that the parent was not called self.assertFalse(parent_mock.open.called) # reset mocks MagicMock.reset_mock(parent_mock) # remove the refresh header info_response.keys.return_value = ["SomeOtherHeaders"] # execute refresh_processor.http_response(request, response) # ensure that the parent was not called self.assertFalse(parent_mock.open.called) # reset mocks MagicMock.reset_mock(parent_mock) # use and invalid http code but include a refresh header response.code = 401 info_response.keys.return_value = ["Refresh"] info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080" # execute refresh_processor.http_response(request, response) # ensure that the parent was not called self.assertFalse(parent_mock.open.called)