def _load_jmx(self, ssl, host, port, jmx_metric):
""" creates a JmxMetric object that holds info about jmx-based metrics """
value_list = []
kerberos_keytab = None
kerberos_principal = None
if logger.isEnabledFor(logging.DEBUG):
logger.debug(str(jmx_metric.property_map))
security_enabled = str(self._get_configuration_value(SECURITY_ENABLED_KEY)).upper() == 'TRUE'
if self.uri_property_keys.kerberos_principal is not None:
kerberos_principal = self._get_configuration_value(
self.uri_property_keys.kerberos_principal)
if kerberos_principal is not None:
# substitute _HOST in kerberos principal with actual fqdn
kerberos_principal = kerberos_principal.replace('_HOST', self.host_name)
if self.uri_property_keys.kerberos_keytab is not None:
kerberos_keytab = self._get_configuration_value(self.uri_property_keys.kerberos_keytab)
if "0.0.0.0" in str(host):
host = self.host_name
for jmx_property_key, jmx_property_value in jmx_metric.property_map.iteritems():
url = "{0}://{1}:{2}/jmx?qry={3}".format(
"https" if ssl else "http", host, str(port), jmx_property_key)
# use a customer header processor that will look for the non-standard
# "Refresh" header and attempt to follow the redirect
response = None
content = ''
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
tmp_dir = Constants.AGENT_TMP_DIR
if tmp_dir is None:
tmp_dir = gettempdir()
kerberos_executable_search_paths = self._get_configuration_value('{{kerberos-env/executable_search_paths}}')
smokeuser = self._get_configuration_value('{{cluster-env/smokeuser}}')
response, error_msg, time_millis = curl_krb_request(tmp_dir, kerberos_keytab, kerberos_principal, url,
"metric_alert", kerberos_executable_search_paths, False, self.get_name(), smokeuser,
connection_timeout=self.curl_connection_timeout, kinit_timer_ms = self.kinit_timeout)
content = response
else:
url_opener = urllib2.build_opener(RefreshHeaderProcessor())
response = url_opener.open(url, timeout=self.connection_timeout)
content = response.read()
except Exception, exception:
if logger.isEnabledFor(logging.DEBUG):
logger.exception("[Alert][{0}] Unable to make a web request: {1}".format(self.get_name(), str(exception)))
finally:
def test_metric_alert_uses_refresh_processor(self, http_response_mock,
http_connection_mock):
"""
Tests that the RefreshHeaderProcessor is correctly chained and called
:param http_response_mock:
:param http_connection_mock:
:return:
"""
http_conn = http_connection_mock.return_value
http_conn.getresponse.return_value = MagicMock(status=200)
http_response_mock.return_value = MagicMock(code=200)
url_opener = urllib2.build_opener(RefreshHeaderProcessor())
response = url_opener.open("http://foo.bar.baz/jmx")
self.assertFalse(response is None)
self.assertTrue(http_conn.request.called)
self.assertTrue(http_conn.getresponse.called)
self.assertTrue(http_response_mock.called)
# now we know that the refresh header is intercepting, reset the mocks
# and try with a METRIC alert
MagicMock.reset_mock(http_response_mock)
MagicMock.reset_mock(http_connection_mock)
definition_json = self._get_metric_alert_definition()
configuration = {
'hdfs-site': {
'dfs.datanode.http.address': 'c6401.ambari.apache.org:80'
}
}
collector = AlertCollector()
cluster_configuration = self.__get_cluster_configuration()
self.__update_cluster_configuration(cluster_configuration,
configuration)
alert = MetricAlert(definition_json, definition_json['source'])
alert.set_helpers(collector, cluster_configuration)
alert.set_cluster("c1", "c6401.ambari.apache.org")
alert.collect()
self.assertFalse(response is None)
self.assertTrue(http_conn.request.called)
self.assertTrue(http_conn.getresponse.called)
self.assertTrue(http_response_mock.called)
def get_value_from_jmx(query, jmx_property, connection_timeout):
response = None
try:
# use a customer header process that will look for the non-standard
# "Refresh" header and attempt to follow the redirect
url_opener = urllib2.build_opener(RefreshHeaderProcessor())
response = url_opener.open(query, timeout=connection_timeout)
data = response.read()
data_dict = json.loads(data)
return data_dict["beans"][0][jmx_property]
finally:
if response is not None:
try:
response.close()
except:
pass
def _load_jmx(self, ssl, host, port, jmx_metric):
""" creates a JmxMetric object that holds info about jmx-based metrics """
value_list = []
if logger.isEnabledFor(logging.DEBUG):
logger.debug(str(jmx_metric.property_map))
for jmx_property_key, jmx_property_value in jmx_metric.property_map.iteritems(
):
url = "{0}://{1}:{2}/jmx?qry={3}".format(
"https" if ssl else "http", host, str(port), jmx_property_key)
# use a customer header processor that will look for the non-standard
# "Refresh" header and attempt to follow the redirect
response = None
try:
url_opener = urllib2.build_opener(RefreshHeaderProcessor())
response = url_opener.open(url, timeout=CONNECTION_TIMEOUT)
content = response.read()
finally:
# explicitely close the connection as we've seen python hold onto these
if response is not None:
try:
response.close()
except:
logger.debug(
"[Alert][{0}] Unable to close JMX URL connection to {1}"
.format(self.get_name(), url))
json_response = json.loads(content)
json_data = json_response['beans'][0]
for attr in jmx_property_value:
if attr not in json_data:
raise Exception(
"Unable to find {0} in JSON from {1} ".format(
attr, url))
value_list.append(json_data[attr])
return value_list
def test_urllib2_refresh_header_processor(self):
from urllib2 import Request
# setup the original request
original_url = "http://foo.bar.baz/jmx?qry=someQuery"
request = Request(original_url)
# ensure that we get back a 200 with a refresh header to redirect us
response = MagicMock(code=200)
info_response = MagicMock()
info_response.keys.return_value = ["Refresh"]
info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080"
response.info.return_value = info_response
# add a mock parent to the refresh processor
parent_mock = MagicMock()
refresh_processor = RefreshHeaderProcessor()
refresh_processor.parent = parent_mock
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was called with the modified URL
parent_mock.open.assert_called_with(
"http://foobar.baz.qux:8080/jmx?qry=someQuery")
# reset mocks
MagicMock.reset_mock(parent_mock)
# alter the refresh header to remove the time value
info_response.getheader.return_value = "url=http://foobar.baz.qux:8443"
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was called with the modified URL
parent_mock.open.assert_called_with(
"http://foobar.baz.qux:8443/jmx?qry=someQuery")
# reset mocks
MagicMock.reset_mock(parent_mock)
# use an invalid refresh header
info_response.getheader.return_value = "http://foobar.baz.qux:8443"
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was not called
self.assertFalse(parent_mock.open.called)
# reset mocks
MagicMock.reset_mock(parent_mock)
# remove the refresh header
info_response.keys.return_value = ["SomeOtherHeaders"]
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was not called
self.assertFalse(parent_mock.open.called)
# reset mocks
MagicMock.reset_mock(parent_mock)
# use and invalid http code but include a refresh header
response.code = 401
info_response.keys.return_value = ["Refresh"]
info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080"
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was not called
self.assertFalse(parent_mock.open.called)
def test_urllib2_refresh_header_processor(self):
from urllib2 import Request
# setup the original request
original_url = "http://foo.bar.baz/jmx?qry=someQuery"
request = Request(original_url)
# ensure that we get back a 200 with a refresh header to redirect us
response = MagicMock(code=200)
info_response = MagicMock()
info_response.keys.return_value = ["Refresh"]
info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080"
response.info.return_value = info_response
# add a mock parent to the refresh processor
parent_mock = MagicMock()
refresh_processor = RefreshHeaderProcessor()
refresh_processor.parent = parent_mock
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was called with the modified URL
parent_mock.open.assert_called_with("http://foobar.baz.qux:8080/jmx?qry=someQuery")
# reset mocks
MagicMock.reset_mock(parent_mock)
# alter the refresh header to remove the time value
info_response.getheader.return_value = "url=http://foobar.baz.qux:8443"
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was called with the modified URL
parent_mock.open.assert_called_with("http://foobar.baz.qux:8443/jmx?qry=someQuery")
# reset mocks
MagicMock.reset_mock(parent_mock)
# use an invalid refresh header
info_response.getheader.return_value = "http://foobar.baz.qux:8443"
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was not called
self.assertFalse(parent_mock.open.called)
# reset mocks
MagicMock.reset_mock(parent_mock)
# remove the refresh header
info_response.keys.return_value = ["SomeOtherHeaders"]
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was not called
self.assertFalse(parent_mock.open.called)
# reset mocks
MagicMock.reset_mock(parent_mock)
# use and invalid http code but include a refresh header
response.code = 401
info_response.keys.return_value = ["Refresh"]
info_response.getheader.return_value = "3; url=http://foobar.baz.qux:8080"
# execute
refresh_processor.http_response(request, response)
# ensure that the parent was not called
self.assertFalse(parent_mock.open.called)