def __init__(self, host, port=8082, user='', password=''):
if user and password:
auth = HTTPBasicAuthHandler()
auth.add_password('Django', '%s:%s' % (host, port), user, password)
self.__opener(auth)
else:
self.__opener = build_opener()
def request(cls, url, start, end, realm='Webtrends Basic Authentication'):
"""Make an authed request to the webtrends API.
Make one attempt to fetch and reload the data. If something fails, it's
the caller's responsibility to retry.
"""
# If start and/or end are date or datetime, convert to string.
if isinstance(start, (date, datetime)):
start = start.strftime('%Ym%md%d')
if isinstance(end, (date, datetime)):
end = end.strftime('%Ym%md%d')
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm,
uri=url,
user=settings.WEBTRENDS_USER,
passwd=settings.WEBTRENDS_PASSWORD)
opener = build_opener(auth_handler)
url = urlparams(url, start_period=start, end_period=end)
try:
# TODO: A wrong username or password results in a recursion depth
# error.
return opener.open(url).read()
except IOError, e:
raise StatsIOError(*e.args)
def __init__(self, url, pysparkle):
self.url = url
self.pysparkle = pysparkle
auth_user = pysparkle.config.get('auth_user')
if auth_user is not None:
auth_pass = pysparkle.config.get('auth_password', '')
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(None, url, auth_user, auth_pass)
opener = build_opener(auth_handler)
install_opener(opener)
def connect(self):
"""Get a handle to a remote connection."""
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=self.url,
uri=self.url,
user=self._user_name,
passwd=self._password)
opener = build_opener(auth_handler)
install_opener(opener)
return self
def get_handler(url):
hdlr = HTTPBasicAuthHandler()
hdlr.add_password("Archives", urlparse(url)[1], LOGIN, PASSWD)
opener = build_opener(hdlr)
install_opener(opener)
req = Request(url)
b64str = encodestring('%s:%s' % (LOGIN, PASSWD))[:-1]
req.add_header('Authorization', 'Basic %s' % b64str)
return req
def connect(self):
"""Get a handle to a remote connection."""
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=self.url,
uri=self.url,
user=self._user_name,
passwd=self._password)
opener = build_opener(auth_handler)
install_opener(opener)
return self
def get_handler(url):
hdlr = HTTPBasicAuthHandler()
hdlr.add_password("Archives",urlparse(url)[1], LOGIN,PASSWD)
opener = build_opener(hdlr)
install_opener(opener)
req = Request(url)
b64str = encodestring('%s:%s' % (LOGIN,PASSWD))[:-1]
req.add_header('Authorization','Basic %s' % b64str)
return req
def get_wsdls(hostname,
username='******',
password='******',
verify=False,
timeout=90,
port=443):
"""Returns the set of all available WSDLs on this server
Used for providing introspection into the available namespaces and WSDLs
dynamically (e.g. when using iPython)
@param hostname: The IP address or hostname of the BIGIP.
@param username: The admin username on the BIGIP.
@param password: The admin password on the BIGIP.
@param verify: When True, performs SSL certificate validation in
Python / urllib2 versions that support it (v2.7.9 and newer)
@param timeout: The time to wait (in seconds) before timing out the connection
to the URL
"""
url = 'https://%s:%s/iControl/iControlPortal.cgi' % (hostname, port)
regex = re.compile(r'/iControl/iControlPortal.cgi\?WSDL=([^"]+)"')
auth_handler = HTTPBasicAuthHandler()
# 10.1.0 has a realm of "BIG-IP"
auth_handler.add_password(uri='https://%s:%s/' % (hostname, port),
user=username,
passwd=password,
realm="BIG-IP")
# 11.3.0 has a realm of "BIG-\IP". I'm not sure exactly when it changed.
auth_handler.add_password(uri='https://%s:%s/' % (hostname, port),
user=username,
passwd=password,
realm="BIG\-IP")
if verify:
opener = build_opener(auth_handler)
else:
opener = build_opener(auth_handler, HTTPSHandlerNoVerify)
try:
result = opener.open(url, timeout=timeout)
except URLError as e:
raise ConnectionError(str(e))
wsdls = {}
for line in result.readlines():
result = regex.search(line.decode())
if result:
namespace, rest = result.groups()[0].split(".", 1)
if namespace not in wsdls:
wsdls[namespace] = []
wsdls[namespace].append(rest)
return wsdls
def _add_basic_auth(self):
auth_handler = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
auth_handler.add_password(
realm=None,
uri=self.host,
user=self.user,
passwd=self.password,
)
install_opener(
build_opener(
auth_handler,
HTTPHandler(debuglevel=self.request_debug_level),
HTTPSHandler(debuglevel=self.request_debug_level),
))
def _add_basic_auth(self):
auth_handler = HTTPBasicAuthHandler(
HTTPPasswordMgrWithDefaultRealm()
)
auth_handler.add_password(
realm=None,
uri=self.host,
user=self.user,
passwd=self.password,
)
install_opener(build_opener(
auth_handler,
HTTPHandler(debuglevel=self.request_debug_level),
HTTPSHandler(debuglevel=self.request_debug_level),
))
def _createUser(self, number):
record = self._records[number]
user = record.uid
authBasic = HTTPBasicAuthHandler(password_mgr=HTTPPasswordMgrWithDefaultRealm())
authBasic.add_password(
realm=None,
uri=self.servers[record.podID]["uri"],
user=user.encode('utf-8'),
passwd=record.password.encode('utf-8'))
authDigest = HTTPDigestAuthHandler(passwd=HTTPPasswordMgrWithDefaultRealm())
authDigest.add_password(
realm=None,
uri=self.servers[record.podID]["uri"],
user=user.encode('utf-8'),
passwd=record.password.encode('utf-8'))
return record, user, {"basic": authBasic, "digest": authDigest, }
def _createUser(self, number):
record = self._records[number]
user = record.uid
authBasic = HTTPBasicAuthHandler()
authBasic.add_password(
realm="Test Realm",
uri=self.server,
user=user.encode('utf-8'),
passwd=record.password.encode('utf-8'))
authDigest = HTTPDigestAuthHandler()
authDigest.add_password(
realm="Test Realm",
uri=self.server,
user=user.encode('utf-8'),
passwd=record.password.encode('utf-8'))
return user, {"basic": authBasic, "digest": authDigest, }
def json_for(cls, period):
"""Return the JSON-formatted WebTrends stats for the given period.
Make one attempt to fetch and reload the data. If something fails, it's
the caller's responsibility to retry.
"""
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=settings.WEBTRENDS_REALM,
uri=settings.WEBTRENDS_WIKI_REPORT_URL,
user=settings.WEBTRENDS_USER,
passwd=settings.WEBTRENDS_PASSWORD)
opener = build_opener(auth_handler)
start, end = period_dates()[period]
url = (settings.WEBTRENDS_WIKI_REPORT_URL +
'&start_period=%s&end_period=%s' % (start, end))
try:
# TODO: A wrong username or password results in a recursion depth
# error.
return opener.open(url).read()
except IOError, e:
raise StatsIOError(*e.args)
def callservice(url, method='post', basicauth=None, **kargs):
from urllib import urlencode
from urllib2 import (urlopen, build_opener, install_opener, HTTPError,
HTTPBasicAuthHandler)
params = ''
if kargs:
# urlencode does not accept non-ascii data
for k, v in kargs.iteritems():
if isinstance(v, unicode):
kargs[k] = v.encode('utf-8')
params = urlencode(kargs)
if not _testing:
# If we need to authenticate
if basicauth is not None:
# Get host from url
host = url[url.find('//') + 2:]
for c in (':', '/', '?', '&'):
if c in host:
host = host[:host.find(c)]
# Install authentication handler
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(basicauth['realm'], host,
basicauth['user'], basicauth['passwd'])
install_opener(build_opener(auth_handler))
try:
if method == 'post':
u = urlopen(url, data=params)
elif method == 'get':
if params:
url = '%s?%s' % (url, params)
u = urlopen(url)
else:
raise ValueError('Unknown method: %s' % method)
except HTTPError, e:
raise ValueError(e.read())
resp = u.read()
return resp
def open_url(url, config, data=None, handlers=None):
"""Attempts to open a connection to a specified URL.
@param url: URL to attempt to open
@param config: SSL context configuration
@type config: Configuration
@param data: HTTP POST data
@type data: str
@param handlers: list of custom urllib2 handlers to add to the request
@type handlers: iterable
@return: tuple (
returned HTTP status code or 0 if an error occurred
returned message or error description
response object)
"""
debuglevel = 1 if config.debug else 0
# Set up handlers for URL opener.
if config.cookie:
cj = config.cookie
else:
cj = cookielib.CookieJar()
# Use a cookie processor that accumulates cookies when redirects occur so
# that an application can redirect for authentication and retain both any
# cookies for the application and the security system (c.f.,
# urllib2.HTTPCookieProcessor which replaces cookies).
cookie_handler = AccumulatingHTTPCookieProcessor(cj)
if not handlers:
handlers = []
handlers.append(cookie_handler)
if config.debug:
http_handler = HTTPHandler(debuglevel=debuglevel)
https_handler = HTTPSContextHandler(config.ssl_context,
debuglevel=debuglevel)
handlers.extend([http_handler, https_handler])
if config.http_basicauth:
# currently only supports http basic auth
auth_handler = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
auth_handler.add_password(realm=None, uri=url,
user=config.httpauth[0],
passwd=config.httpauth[1])
handlers.append(auth_handler)
# Explicitly remove proxy handling if the host is one listed in the value of
# the no_proxy environment variable because urllib2 does use proxy settings
# set via http_proxy and https_proxy, but does not take the no_proxy value
# into account.
if not _should_use_proxy(url, config.no_proxy):
handlers.append(urllib2.ProxyHandler({}))
log.debug("Not using proxy")
elif config.proxies:
handlers.append(urllib2.ProxyHandler(config.proxies))
log.debug("Configuring proxies: %s" % config.proxies)
opener = build_opener(*handlers, ssl_context=config.ssl_context)
headers = config.headers
if headers is None:
headers = {}
request = urllib2.Request(url, data, headers)
# Open the URL and check the response.
return_code = 0
return_message = ''
response = None
try:
response = opener.open(request)
return_message = response.msg
return_code = response.code
if log.isEnabledFor(logging.DEBUG):
for index, cookie in enumerate(cj):
log.debug("%s : %s", index, cookie)
except urllib2.HTTPError, exc:
return_code = exc.code
return_message = "Error: %s" % exc.msg
if log.isEnabledFor(logging.DEBUG):
log.debug("%s %s", exc.code, exc.msg)
from urllib2 import HTTPBasicAuthHandler, build_opener, install_opener
#Database settings
database_connect = "dbname='ridprod'"
kv1_database_connect = "dbname='kv1tmp'"
iff_database_connect = "dbname='ifftmp'"
pool_generation_enabled = False
import_arriva_trains = False
# Subscribe to https://ndovloket.nl/aanvragen/
#
# username and password can be found here:
# https://groups.google.com/forum/#!topic/ndovloket-meldingen/IxEPpXds_Qo
#NDOVLoket settings
ndovloket_url = "data.ndovloket.nl"
ndovloket_user = None
ndovloket_password = None
if not ndovloket_user or not ndovloket_password:
print("Subscribe to https://ndovloket.nl/aanvragen/\n\nusername and password can be found here:\nhttps://groups.google.com/forum/#!topic/ndovloket-meldingen/IxEPpXds_Qo")
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=ndovloket_url,
uri=ndovloket_url,
user=ndovloket_user,
passwd=ndovloket_password)
opener = build_opener(auth_handler)
install_opener(opener)
def open_url(url, config, data=None, handlers=None):
"""Attempts to open a connection to a specified URL.
@param url: URL to attempt to open
@param config: SSL context configuration
@type config: Configuration
@param data: HTTP POST data
@type data: str
@param handlers: list of custom urllib2 handlers to add to the request
@type handlers: iterable
@return: tuple (
returned HTTP status code or 0 if an error occurred
returned message or error description
response object)
"""
debuglevel = 1 if config.debug else 0
# Set up handlers for URL opener.
if config.cookie:
cj = config.cookie
else:
cj = cookielib.CookieJar()
# Use a cookie processor that accumulates cookies when redirects occur so
# that an application can redirect for authentication and retain both any
# cookies for the application and the security system (c.f.,
# urllib2.HTTPCookieProcessor which replaces cookies).
cookie_handler = AccumulatingHTTPCookieProcessor(cj)
if not handlers:
handlers = []
handlers.append(cookie_handler)
if config.debug:
http_handler = HTTPHandler(debuglevel=debuglevel)
https_handler = HTTPSContextHandler(config.ssl_context,
debuglevel=debuglevel)
handlers.extend([http_handler, https_handler])
if config.http_basicauth:
# currently only supports http basic auth
auth_handler = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
auth_handler.add_password(realm=None, uri=url,
user=config.httpauth[0],
passwd=config.httpauth[1])
handlers.append(auth_handler)
# Explicitly remove proxy handling if the host is one listed in the value of
# the no_proxy environment variable because urllib2 does use proxy settings
# set via http_proxy and https_proxy, but does not take the no_proxy value
# into account.
if not _should_use_proxy(url, config.no_proxy):
handlers.append(urllib2.ProxyHandler({}))
log.debug("Not using proxy")
elif config.proxies:
handlers.append(urllib2.ProxyHandler(config.proxies))
log.debug("Configuring proxies: %s" % config.proxies)
opener = build_opener(*handlers, ssl_context=config.ssl_context)
headers = config.headers
if headers is None:
headers = {}
request = urllib2.Request(url, data, headers)
# Open the URL and check the response.
return_code = 0
return_message = ''
response = None
try:
response = opener.open(request)
return_message = response.msg
return_code = response.code
if log.isEnabledFor(logging.DEBUG):
for index, cookie in enumerate(cj):
log.debug("%s : %s", index, cookie)
except urllib2.HTTPError, exc:
return_code = exc.code
return_message = "Error: %s" % exc.msg
if log.isEnabledFor(logging.DEBUG):
log.debug("%s %s", exc.code, exc.msg)
def _post_url(self, event, url=None):
"Posts a URL to delicious.com"
title = self._get_title(url)
con_re = re.compile(r'!n=|!')
connection_body = con_re.split(event.sender['connection'])
if len(connection_body) == 1:
connection_body.append(event.sender['connection'])
ip_re = re.compile(r'\.IP$|unaffiliated')
if ip_re.search(connection_body[1]) != None:
connection_body[1] = ''
if ibid.sources[event.source].type == 'jabber':
obfusc_conn = ''
obfusc_chan = event.channel.replace('@', '^')
else:
at_re = re.compile(r'@\S+?\.')
obfusc_conn = at_re.sub('^', connection_body[1])
obfusc_chan = at_re.sub('^', event.channel)
tags = u' '.join((event.sender['nick'], obfusc_conn, obfusc_chan,
event.source))
data = {
'url' : url.encode('utf-8'),
'description' : title.encode('utf-8'),
'tags' : tags.encode('utf-8'),
'replace' : 'yes',
'dt' : event.time.strftime('%Y-%m-%dT%H:%M:%SZ'),
'extended' : event.message['raw'].encode('utf-8'),
}
if self.service.lower() == 'delicious':
service = ('del.icio.us API', 'https://api.del.icio.us')
elif self.service.lower() == 'faves':
service = ('Faves', 'https://secure.faves.com')
else:
log.error(u'Unknown social bookmarking service: %s', self.service)
return
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(service[0], service[1],
self.username, self.password)
opener = build_opener(auth_handler)
posturl = service[1] + '/v1/posts/add?' + urlencode(data)
try:
resp = opener.open(posturl).read()
if 'done' in resp:
log.debug(u"Posted url '%s' to %s, posted in %s on %s "
u"by %s/%i (%s)",
url, self.service, event.channel, event.source,
event.account, event.identity,
event.sender['connection'])
else:
log.error(u"Error posting url '%s' to %s: %s",
url, self.service, resp)
except HTTPError, e:
if e.code == 401:
log.error(u"Incorrect password for %s, couldn't post",
self.service)
def _post_url(self, event, url=None):
"Posts a URL to delicious.com"
title = self._get_title(url)
con_re = re.compile(r'!n=|!')
connection_body = con_re.split(event.sender['connection'])
if len(connection_body) == 1:
connection_body.append(event.sender['connection'])
ip_re = re.compile(r'\.IP$|unaffiliated')
if ip_re.search(connection_body[1]) != None:
connection_body[1] = ''
if ibid.sources[event.source].type == 'jabber':
obfusc_conn = ''
obfusc_chan = event.channel.replace('@', '^')
else:
at_re = re.compile(r'@\S+?\.')
obfusc_conn = at_re.sub('^', connection_body[1])
obfusc_chan = at_re.sub('^', event.channel)
tags = u' '.join(
(event.sender['nick'], obfusc_conn, obfusc_chan, event.source))
data = {
'url': url.encode('utf-8'),
'description': title.encode('utf-8'),
'tags': tags.encode('utf-8'),
'replace': 'yes',
'dt': event.time.strftime('%Y-%m-%dT%H:%M:%SZ'),
'extended': event.message['raw'].encode('utf-8'),
}
if self.service.lower() == 'delicious':
service = ('del.icio.us API', 'https://api.del.icio.us')
elif self.service.lower() == 'faves':
service = ('Faves', 'https://secure.faves.com')
else:
log.error(u'Unknown social bookmarking service: %s', self.service)
return
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(service[0], service[1], self.username,
self.password)
opener = build_opener(auth_handler)
posturl = service[1] + '/v1/posts/add?' + urlencode(data)
try:
resp = opener.open(posturl).read()
if 'done' in resp:
log.debug(
u"Posted url '%s' to %s, posted in %s on %s "
u"by %s/%i (%s)", url, self.service, event.channel,
event.source, event.account, event.identity,
event.sender['connection'])
else:
log.error(u"Error posting url '%s' to %s: %s", url,
self.service, resp)
except HTTPError, e:
if e.code == 401:
log.error(u"Incorrect password for %s, couldn't post",
self.service)