def determine_auth_type(cls, url, proxies=None, timeout=None, cert=None, logger=None):
"""
Determine the authentication type that is appropriate to authenticate to the given web-server.
Argument:
url -- The url to connect to. This object ought to be an instance derived from using urlparse
proxies -- The proxies to use
timeout -- The amount of time to quit waiting on a connection
cert -- A tuple representing the certificate to use
logger -- The logger object to use for logging
"""
# Perform a request to the URL and see what authentication method is required
# Make the client
http = requests.get(url.geturl(), proxies=proxies, timeout=timeout, cert=cert, verify=False)
try:
auth_header = http.headers['WWW-Authenticate']
if auth_header is not None:
m = re.search('^([a-zA-Z0-9]+) ', auth_header)
auth_type = m.group(1)
return auth_type.lower()
except Exception:
if logger:
logger.exception("Unable to determine authentication type")
def determine_auth_type(cls,
url,
proxies=None,
timeout=None,
cert=None,
logger=None):
"""
Determine the authentication type that is appropriate to authenticate to the given web-server.
Argument:
url -- The url to connect to. This object ought to be an instance derived from using urlparse
proxies -- The proxies to use
timeout -- The amount of time to quit waiting on a connection
cert -- A tuple representing the certificate to use
logger -- The logger object to use for logging
"""
# Perform a request to the URL and see what authentication method is required
try:
# Make the GET
http = requests.get(url.geturl(),
proxies=proxies,
timeout=timeout,
cert=cert,
verify=False)
# Find the authentication header irrespective of case
auth_header_value = None
for header, value in http.headers.items():
if header.lower() == 'www-authenticate':
auth_header_value = value
break
# Determine if the authentication header is present and use it to determine the authentication type
if auth_header_value is not None:
# Handle the pesky cases where a comma separated value is provided in the header for NTLM negotiation (like "negotiate, ntlm")
if 'ntlm' in auth_header_value.lower():
return cls.HTTP_AUTH_NTLM
# Otherwise, check the HTTP header for the authentication header
m = re.search('^([a-zA-Z0-9]+)', auth_header_value)
auth_type = m.group(1)
return auth_type.lower()
# No authentication header is present
else:
if logger:
logger.warn(
"Unable to determine authentication type (no www-authenticate header); will default to basic authentication"
)
return cls.HTTP_AUTH_NONE
except Exception:
if logger:
logger.exception("Unable to determine authentication type")
def test_is_exception_for_timeout(self):
try:
r = requests.get('https://192.168.30.23/')
except requests.exceptions.ConnectionError as e:
if not WebPing.isExceptionForTimeout(e):
print(e)
self.assertTrue(WebPing.isExceptionForTimeout(e))
def determine_auth_type(cls, url, proxies=None, timeout=None, cert=None, logger=None):
"""
Determine the authentication type that is appropriate to authenticate to the given
web-server.
Argument:
url -- The url to connect to. This object ought to be an instance derived from using urlparse
proxies -- The proxies to use
timeout -- The amount of time to quit waiting on a connection
cert -- A tuple representing the certificate to use
logger -- The logger object to use for logging
"""
# Perform a request to the URL and see what authentication method is required
try:
# Make the GET
http = requests.get(url.geturl(), proxies=proxies, timeout=timeout, cert=cert,
verify=False)
# Find the authentication header irrespective of case
auth_header_value = None
for header, value in http.headers.items():
if header.lower() == 'www-authenticate':
auth_header_value = value
break
# Determine if the authentication header is present and use it to determine the
# authentication type
if auth_header_value is not None:
# Handle the pesky cases where a comma separated value is provided in the header
# for NTLM negotiation (like "negotiate, ntlm")
if 'ntlm' in auth_header_value.lower():
return cls.HTTP_AUTH_NTLM
# Otherwise, check the HTTP header for the authentication header
m = re.search('^([a-zA-Z0-9]+)', auth_header_value)
auth_type = m.group(1)
return auth_type.lower()
# No authentication header is present
else:
if logger:
logger.warn("Unable to determine authentication type (no www-authenticate header); will default to basic authentication")
return cls.HTTP_AUTH_NONE
except Exception:
if logger:
logger.exception("Unable to determine authentication type")
def ping(cls,
url,
username=None,
password=None,
timeout=30,
proxy_type=None,
proxy_server=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
client_certificate=None,
client_certificate_key=None,
user_agent=None,
logger=None,
should_contain_string=None):
"""
Perform a ping to a website. Returns a WebPing.Result instance.
Argument:
url -- The url to connect to. This object ought to be an instance derived from using urlparse.
username -- The password to use for authentication
password -- The username to use for authentication
timeout -- The amount of time to quit waiting on a connection.
proxy_type -- The type of the proxy server (must be one of: socks4, socks5, http)
proxy_server -- The proxy server to use.
proxy_port -- The port on the proxy server to use.
proxy_user -- The proxy server to use.
proxy_password -- The port on the proxy server to use.
client_certificate -- The path to the client certificate to use.
client_certificate_key -- The path to the client key to use.
user_agent -- The string to use for the user-agent
logger -- The logger object to use for logging
should_contain_string -- A string that is expected in the response
"""
if logger:
logger.info('Performing ping, url="%s"', url.geturl())
# Determine which type of proxy is to be used (if any)
resolved_proxy_type = cls.resolve_proxy_type(proxy_type, logger=logger)
# Set should_contain_string to none if it is blank since this means it really doesn't have
# a value
if should_contain_string is not None and len(
should_contain_string.strip()) == 0:
should_contain_string = None
# Make sure that a timeout is not None since that is infinite
if timeout is None:
timeout = 30
# Setup the proxy info if so configured
proxies = {}
if resolved_proxy_type is not None and proxy_server is not None and len(
proxy_server.strip()) > 0:
if proxy_type == "http":
# Use the username and password if provided
if proxy_password is not None and proxy_user is not None:
proxies = {
"http":
"http://" + proxy_user + ":" + proxy_password + "@" +
proxy_server + ":" + str(proxy_port),
"https":
"http://" + proxy_user + ":" + proxy_password + "@" +
proxy_server + ":" + str(proxy_port)
}
else:
proxies = {
"http":
"http://" + proxy_server + ":" + str(proxy_port),
"https":
"http://" + proxy_server + ":" + str(proxy_port)
}
else:
socks.setdefaultproxy(resolved_proxy_type, proxy_server,
proxy_port)
socket.socket = socks.socksocket
else:
# No proxy is being used
pass
# Setup the client certificate parameter
if client_certificate is not None and client_certificate_key is not None:
cert = (client_certificate, client_certificate_key)
elif client_certificate is not None:
cert = client_certificate
else:
cert = None
if logger and cert is not None:
logger.debug("Using client certificate %s", cert)
request_time = 0
response_code = 0
response_md5 = None
response_sha224 = None
timed_out = False
response_size = None
has_expected_string = None
# Setup the headers as necessary
headers = {}
if user_agent is not None:
if logger:
logger.debug("Setting user-agent=%s", user_agent)
headers['User-Agent'] = user_agent
# Make an auth object if necessary
auth = None
auth_type = None
if username is not None and password is not None:
# Determine the auth type
auth_type = cls.determine_auth_type(url,
proxies=proxies,
timeout=timeout,
cert=cert,
logger=logger)
# Don't allow the use of NTLM on a host in FIPS mode since NTLM uses MD4 which is a
# weak algorithm
if auth_type == cls.HTTP_AUTH_NTLM and cls.is_fips_mode():
if logger:
logger.warn(
"Authentication type was automatically identified but will not be used since it uses a weak hash algorithm which is not allowed on this host since it is running in FIPS mode; auth_type=%s",
auth_type)
auth_type = cls.HTTP_AUTH_NONE
# The authentication type could not be determined. However, we know that
# authentication is required since a username and password was provided.
# Default to HTTP basic authentication.
elif auth_type == cls.HTTP_AUTH_NONE:
auth_type = cls.HTTP_AUTH_BASIC
if logger:
logger.info(
"Authentication type could not be automatically discovered; auth_type=%s",
auth_type)
elif logger is not None:
logger.debug("Discovered auth_type=%s", auth_type)
# Get the authentication class for request
auth = cls.create_auth_for_request(auth_type, username, password,
logger)
try:
# Perform the request
with Timer() as timer:
# Make the client
http = requests.get(url.geturl(),
proxies=proxies,
timeout=timeout,
cert=cert,
verify=False,
auth=auth,
headers=headers)
# Get the hash of the content
if not cls.is_fips_mode():
response_md5 = hashlib.md5(http.text).hexdigest()
response_sha224 = hashlib.sha224(http.text).hexdigest()
# Determine if the expected string is in the content
if should_contain_string is not None:
has_expected_string = should_contain_string in http.text
# Get the size of the content
response_size = len(http.text)
response_code = http.status_code
request_time = timer.msecs
# Handle time outs
except requests.exceptions.Timeout:
# Note that the connection timed out
timed_out = True
except requests.exceptions.SSLError as e:
if logger:
logger.error(
"An SSL exception was thrown when executing a web request against url=%s: "
+ str(e), url.geturl())
except requests.exceptions.ConnectionError as e:
if e.args is not None and len(e.args) > 0 and hasattr(
e.args[0], 'reason') and hasattr(
e.args[0].reason,
'errno') and e.args[0].reason.errno in [
60, 61, 10060, 10061
]:
timed_out = True
elif logger:
logger.exception(
"A connection exception was thrown when executing a web request against url=%s, this can happen if the domain name, IP address is invalid or if network connectivity is down or blocked by a firewall; see help_url=http://lukemurphey.net/projects/splunk-website-monitoring/wiki/Troubleshooting",
url.geturl())
except socks.GeneralProxyError:
# This may be thrown if the user configured the proxy settings incorrectly
if logger:
logger.exception(
"An error occurred when attempting to communicate with the proxy for url=%s",
url.geturl())
except Exception as e:
if logger:
logger.exception(
"A general exception was thrown when executing a web request for url=%s",
url.geturl())
# Finally, return the result
return cls.Result(request_time, response_code, timed_out, url.geturl(),
response_size, response_md5, response_sha224,
has_expected_string)
def ping(cls, url, username=None, password=None, timeout=30, proxy_type=None,
proxy_server=None, proxy_port=None, proxy_user=None, proxy_password=None, proxy_ignore=None,
client_certificate=None, client_certificate_key=None, user_agent=None,
logger=None, should_contain_string=None):
"""
Perform a ping to a website. Returns a WebPing.Result instance.
Argument:
url -- The url to connect to. This object ought to be an instance derived from using urlparse.
username -- The password to use for authentication
password -- The username to use for authentication
timeout -- The amount of time to quit waiting on a connection.
proxy_type -- The type of the proxy server (must be one of: socks4, socks5, http)
proxy_server -- The proxy server to use.
proxy_port -- The port on the proxy server to use.
proxy_user -- The proxy server to use.
proxy_password -- The port on the proxy server to use.
proxy_ignore -- The list of domains to not use the proxy server for.
client_certificate -- The path to the client certificate to use.
client_certificate_key -- The path to the client key to use.
user_agent -- The string to use for the user-agent
logger -- The logger object to use for logging
should_contain_string -- A string that is expected in the response
"""
if logger:
logger.info('Performing ping, url="%s"', url.geturl())
# Disable the use of the proxy variables
if proxy_ignore is not None:
os.environ['NO_PROXY'] = proxy_ignore
if logger:
logger.debug('Proxies discovered from the environment, proxies="%r"', urllib.getproxies())
# Determine which type of proxy is to be used (if any)
resolved_proxy_type = cls.resolve_proxy_type(proxy_type, logger=logger)
# Set should_contain_string to none if it is blank since this means it really doesn't have
# a value
if should_contain_string is not None and len(should_contain_string.strip()) == 0:
should_contain_string = None
# Make sure that a timeout is not None since that is infinite
if timeout is None:
timeout = 30
# Setup the proxy info if so configured
proxies = {}
if resolved_proxy_type is not None and proxy_server is not None and len(proxy_server.strip()) > 0:
if proxy_type == "http":
# Use the username and password if provided
if proxy_password is not None and proxy_user is not None:
proxies = {
"http": "http://" + proxy_user + ":" + proxy_password + "@" + proxy_server + ":" + str(proxy_port),
"https": "http://" + proxy_user + ":" + proxy_password + "@" + proxy_server + ":" + str(proxy_port)
}
else:
proxies = {
"http": "http://" + proxy_server + ":" + str(proxy_port),
"https": "http://" + proxy_server + ":" + str(proxy_port)
}
else:
socks.setdefaultproxy(resolved_proxy_type, proxy_server, int(proxy_port))
socket.socket = socks.socksocket
if logger:
logger.debug("Using socks proxy server=%s, port=%s", proxy_server, proxy_port)
else:
# No proxy is being used
pass
# Setup the client certificate parameter
if client_certificate is not None and client_certificate_key is not None:
cert = (client_certificate, client_certificate_key)
elif client_certificate is not None:
cert = client_certificate
else:
cert = None
if logger and cert is not None:
logger.debug("Using client certificate %s", cert)
request_time = 0
response_code = 0
response_md5 = None
response_sha224 = None
timed_out = False
response_size = None
has_expected_string = None
# Setup the headers as necessary
headers = {}
if user_agent is not None:
if logger:
logger.debug("Setting user-agent=%s", user_agent)
headers['User-Agent'] = user_agent
# Make an auth object if necessary
auth = None
auth_type = None
if username is not None and password is not None:
# Determine the auth type
auth_type = cls.determine_auth_type(url, proxies=proxies, timeout=timeout, cert=cert,
logger=logger)
# Don't allow the use of NTLM on a host in FIPS mode since NTLM uses MD4 which is a
# weak algorithm
if auth_type == cls.HTTP_AUTH_NTLM and cls.is_fips_mode():
if logger:
logger.warn("Authentication type was automatically identified but will not be used since it uses a weak hash algorithm which is not allowed on this host since it is running in FIPS mode; auth_type=%s", auth_type)
auth_type = cls.HTTP_AUTH_NONE
# The authentication type could not be determined. However, we know that
# authentication is required since a username and password was provided.
# Default to HTTP basic authentication.
elif auth_type == cls.HTTP_AUTH_NONE:
auth_type = cls.HTTP_AUTH_BASIC
if logger:
logger.info("Authentication type could not be automatically discovered; auth_type=%s", auth_type)
elif logger is not None:
logger.debug("Discovered auth_type=%s", auth_type)
# Get the authentication class for request
auth = cls.create_auth_for_request(auth_type, username, password, logger)
try:
# Perform the request
with Timer() as timer:
# Make the client
http = requests.get(url.geturl(), proxies=proxies, timeout=timeout, cert=cert, verify=False, auth=auth, headers=headers)
# Get the hash of the content
if not cls.is_fips_mode():
response_md5 = hashlib.md5(http.text).hexdigest()
response_sha224 = hashlib.sha224(http.text).hexdigest()
# Determine if the expected string is in the content
if should_contain_string is not None:
has_expected_string = should_contain_string in http.text
# Get the size of the content
response_size = len(http.text)
response_code = http.status_code
request_time = timer.msecs
# Handle time outs
except requests.exceptions.Timeout:
# Note that the connection timed out
timed_out = True
except requests.exceptions.SSLError as e:
if logger:
logger.error("An SSL exception was thrown when executing a web request against url=%s: " + str(e), url.geturl())
except requests.exceptions.ConnectionError as e:
timed_out = WebPing.isExceptionForTimeout(e)
if not timed_out and logger:
logger.exception("A connection exception was thrown when executing a web request against url=%s, this can happen if the domain name, IP address is invalid or if network connectivity is down or blocked by a firewall; see help_url=http://lukemurphey.net/projects/splunk-website-monitoring/wiki/Troubleshooting", url.geturl())
except socks.GeneralProxyError:
# This may be thrown if the user configured the proxy settings incorrectly
if logger:
logger.exception("An error occurred when attempting to communicate with the proxy for url=%s", url.geturl())
except Exception as e:
if logger:
logger.exception("A general exception was thrown when executing a web request for url=%s", url.geturl())
# Finally, return the result
return cls.Result(request_time, response_code, timed_out, url.geturl(), response_size, response_md5, response_sha224, has_expected_string)
def ping(cls, url, timeout=30, proxy_type=None, proxy_server=None, proxy_port=None, proxy_user=None, proxy_password=None, client_certificate=None, client_certificate_key=None, logger=None):
"""
Perform a ping to a website. Returns a WebPing.Result instance.
Argument:
url -- The url to connect to. This object ought to be an instance derived from using urlparse.
timeout -- The amount of time to quit waiting on a connection.
proxy_type -- The type of the proxy server (must be one of: socks4, socks5, http)
proxy_server -- The proxy server to use.
proxy_port -- The port on the proxy server to use.
proxy_user -- The proxy server to use.
proxy_password -- The port on the proxy server to use.
client_certificate -- The path to the client certificate to use.
client_certificate_key -- The path to the client key to use.
logger -- The logger object to use for logging
"""
if logger:
logger.debug('Performing ping, url="%s"', url.geturl())
# Determine which type of proxy is to be used (if any)
resolved_proxy_type = cls.resolve_proxy_type(proxy_type, logger=logger)
# Make sure that a timeout is not None since that is infinite
if timeout is None:
timeout = 30
# Setup the proxy info if so configured
proxies = {}
if resolved_proxy_type is not None and proxy_server is not None and len(proxy_server.strip()) > 0:
if proxy_type == "http":
# Use the username and password if provided
if proxy_password is not None and proxy_user is not None:
proxies = {
"http": "http://" + proxy_user + ":" + proxy_password + "@" + proxy_server + ":" + str(proxy_port),
"https": "http://" + proxy_user + ":" + proxy_password + "@" + proxy_server + ":" + str(proxy_port)
}
else:
proxies = {
"http": "http://" + proxy_server + ":" + str(proxy_port),
"https": "http://" + proxy_server + ":" + str(proxy_port)
}
else:
socks.setdefaultproxy(resolved_proxy_type, proxy_server, proxy_port)
socket.socket = socks.socksocket
else:
# No proxy is being used
pass
# Setup the client certificate parameter
if client_certificate is not None and client_certificate_key is not None:
cert = (client_certificate, client_certificate_key)
elif client_certificate is not None:
cert = client_certificate
else:
cert = None
if logger:
logger.debug("Using client certificate %s", cert)
request_time = 0
response_code = 0
response_md5 = None
response_sha224 = None
timed_out = False
response_size = None
try:
# Perform the request
with Timer() as timer:
# Make the client
http = requests.get(url.geturl(), proxies=proxies, timeout=timeout, cert=cert)
# Get the hash of the content
response_md5 = hashlib.md5(http.text).hexdigest()
response_sha224 = hashlib.sha224(http.text).hexdigest()
# Get the size of the content
response_size = len(http.text)
response_code = http.status_code
request_time = timer.msecs
# Handle time outs
except requests.exceptions.Timeout:
# Note that the connection timed out
timed_out = True
except requests.exceptions.ConnectionError as e:
if e.args is not None and len(e.args) > 0 and hasattr(e.args[0], 'reason') and hasattr(e.args[0].reason, 'errno') and e.args[0].reason.errno in [60, 61]:
timed_out = True
elif logger:
logger.exception("A connection exception was thrown when executing a web request")
except socks.GeneralProxyError:
# This may be thrown if the user configured the proxy settings incorrectly
if logger:
logger.exception("An error occurred when attempting to communicate with the proxy")
except Exception as e:
if logger:
logger.exception("A general exception was thrown when executing a web request")
# Finally, return the result
return cls.Result(request_time, response_code, timed_out, url.geturl(), response_size, response_md5, response_sha224)
def ping(cls, url, username=None, password=None, timeout=30, proxy_type=None, proxy_server=None, proxy_port=None, proxy_user=None, proxy_password=None, client_certificate=None, client_certificate_key=None, logger=None):
"""
Perform a ping to a website. Returns a WebPing.Result instance.
Argument:
url -- The url to connect to. This object ought to be an instance derived from using urlparse.
username -- The password to use for authentication
password -- The username to use for authentication
timeout -- The amount of time to quit waiting on a connection.
proxy_type -- The type of the proxy server (must be one of: socks4, socks5, http)
proxy_server -- The proxy server to use.
proxy_port -- The port on the proxy server to use.
proxy_user -- The proxy server to use.
proxy_password -- The port on the proxy server to use.
client_certificate -- The path to the client certificate to use.
client_certificate_key -- The path to the client key to use.
logger -- The logger object to use for logging
"""
if logger:
logger.info('Performing ping, url="%s"', url.geturl())
# Determine which type of proxy is to be used (if any)
resolved_proxy_type = cls.resolve_proxy_type(proxy_type, logger=logger)
# Make sure that a timeout is not None since that is infinite
if timeout is None:
timeout = 30
# Setup the proxy info if so configured
proxies = {}
if resolved_proxy_type is not None and proxy_server is not None and len(proxy_server.strip()) > 0:
if proxy_type == "http":
# Use the username and password if provided
if proxy_password is not None and proxy_user is not None:
proxies = {
"http": "http://" + proxy_user + ":" + proxy_password + "@" + proxy_server + ":" + str(proxy_port),
"https": "http://" + proxy_user + ":" + proxy_password + "@" + proxy_server + ":" + str(proxy_port)
}
else:
proxies = {
"http": "http://" + proxy_server + ":" + str(proxy_port),
"https": "http://" + proxy_server + ":" + str(proxy_port)
}
else:
socks.setdefaultproxy(resolved_proxy_type, proxy_server, proxy_port)
socket.socket = socks.socksocket
else:
# No proxy is being used
pass
# Setup the client certificate parameter
if client_certificate is not None and client_certificate_key is not None:
cert = (client_certificate, client_certificate_key)
elif client_certificate is not None:
cert = client_certificate
else:
cert = None
if logger:
logger.debug("Using client certificate %s", cert)
request_time = 0
response_code = 0
response_md5 = None
response_sha224 = None
timed_out = False
response_size = None
# Make an auth object if necessary
auth = None
auth_type = None
if username is not None and password is not None:
# Determine the auth type
auth_type = cls.determine_auth_type(url, proxies=proxies, timeout=timeout, cert=cert, logger=logger)
# Get the authentication class for request
auth = cls.create_auth_for_request(auth_type, username, password, logger)
if logger is not None:
logger.debug("Discovered auth_type=%s", auth_type)
try:
# Perform the request
with Timer() as timer:
# Make the client
http = requests.get(url.geturl(), proxies=proxies, timeout=timeout, cert=cert, verify=False, auth=auth)
# Get the hash of the content
response_md5 = hashlib.md5(http.text).hexdigest()
response_sha224 = hashlib.sha224(http.text).hexdigest()
# Get the size of the content
response_size = len(http.text)
response_code = http.status_code
request_time = timer.msecs
# Handle time outs
except requests.exceptions.Timeout:
# Note that the connection timed out
timed_out = True
except requests.exceptions.SSLError as e:
if logger:
logger.error("An SSL exception was thrown when executing a web request: " + str(e))
except requests.exceptions.ConnectionError as e:
if e.args is not None and len(e.args) > 0 and hasattr(e.args[0], 'reason') and hasattr(e.args[0].reason, 'errno') and e.args[0].reason.errno in [60, 61]:
timed_out = True
elif logger:
logger.exception("A connection exception was thrown when executing a web request")
except socks.GeneralProxyError:
# This may be thrown if the user configured the proxy settings incorrectly
if logger:
logger.exception("An error occurred when attempting to communicate with the proxy")
except Exception as e:
if logger:
logger.exception("A general exception was thrown when executing a web request")
# Finally, return the result
return cls.Result(request_time, response_code, timed_out, url.geturl(), response_size, response_md5, response_sha224)