def fetch(method, uri, params_prefix=None, **params):
"""Fetch the given uri and return the contents of the response."""
params = urlencode(_prepare_params(params, params_prefix))
binary_params = params.encode('ASCII')
# build the HTTP request
url = "https://%s/%s.xml" % (CHALLONGE_API_URL, uri)
req = Request(url, binary_params)
req.get_method = lambda: method
# use basic authentication
user, api_key = get_credentials()
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key
)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except HTTPError as e:
if e.code != 422:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def create_url_opener(cookie_file_path, domain):
"""Load username and password from .gitcookies and return a URL opener with
an authentication handler."""
# Load authentication credentials
credentials = load_auth_credentials(cookie_file_path)
username, password = credentials[domain]
# Create URL opener with authentication handler
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(domain, domain, username, password)
return build_opener(auth_handler)
def retry_http_basic_auth(self, host, req, realm):
# don't retry if auth failed
if req.get_header(self.auth_header, None) is not None:
return None
return HTTPBasicAuthHandler.retry_http_basic_auth(self, host, req, realm)
def opener_for_url_prefix(
url_prefix, username=None, password=None, cache_dict=None
):
if cache_dict is not None:
cache_key = (url_prefix, username, password)
try:
return cache_dict[cache_key]
except KeyError:
pass
if username or password:
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Open Amiga Game Database",
uri="{0}".format(url_prefix),
user=username,
passwd=password,
)
auth_handler.add_password(
realm="OpenRetro",
uri="{0}".format(url_prefix),
user=username,
passwd=password,
)
opener = build_opener(auth_handler)
else:
opener = build_opener()
if cache_dict is not None:
cache_key = (url_prefix, username, password)
cache_dict[cache_key] = opener
return opener
def fetch(method, uri, params_prefix=None, **params):
"""Fetch the given uri and return the contents of the response."""
params = urlencode(_prepare_params(params, params_prefix))
binary_params = params.encode('ASCII')
# build the HTTP request
url = "https://%s/%s.xml" % (CHALLONGE_API_URL, uri)
req = Request(url, binary_params)
req.get_method = lambda: method
# use basic authentication
user, api_key = get_credentials()
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except HTTPError as e:
if e.code != 422:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def _register_agent(self):
register_name = self.app.config.get('PSDASH_REGISTER_AS')
if not register_name:
register_name = socket.gethostname()
url_args = {
'name': register_name,
'port': self.app.config.get('PSDASH_PORT', self.DEFAULT_PORT),
}
register_url = '%s/register?%s' % (
self.app.config['PSDASH_REGISTER_TO'], urllib.urlencode(url_args))
if 'PSDASH_AUTH_USERNAME' in self.app.config and 'PSDASH_AUTH_PASSWORD' in self.app.config:
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm='psDash login required',
uri=register_url,
user=self.app.config['PSDASH_AUTH_USERNAME'],
passwd=self.app.config['PSDASH_AUTH_PASSWORD'])
opener = build_opener(auth_handler)
install_opener(opener)
try:
urlopen(register_url)
except HTTPError as e:
logger.error('Failed to register agent to "%s": %s', register_url,
e)
def POST(url, args={}, cred=None):
"""do http post
url is the URL you want
args is a dict of cgi args
cred is ( host, realm, username, password )
"""
auth_handler = None
arg_string = ''
if cred is not None:
(host, realm, username, password) = cred
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm, host, username, password)
if auth_handler:
opener = build_opener(cookie_processor, auth_handler)
else:
opener = build_opener(cookie_processor)
install_opener(opener)
print("URL %s" % url)
data = urlencode(args)
req = Request(url, data)
f = urlopen(req)
return f
def get_auth_handler(cls, feed_url, username, password, logger=None):
"""
Create a URL that will perform authentication for the given feed.
Arguments:
feed_url -- The URL of the feed to retrieve (as a string)
username -- The username to use when authenticating
password -- The password to use when authenticating
"""
realm, auth_type = cls.get_realm_and_auth_type(feed_url, username, password, logger)
# Make the associated auth handler
if auth_type == None:
return None
elif auth_type.lower() == "basic":
auth_handler = HTTPBasicAuthHandler()
else:
auth_handler = HTTPDigestAuthHandler()
# Set the password
auth_handler.add_password(realm=realm,
uri=feed_url,
user=username,
passwd=password)
return auth_handler
def POST(url, args={}, cred=None):
"""do http post
url is the URL you want
args is a dict of cgi args
cred is ( host, realm, username, password )
"""
auth_handler = None
arg_string = ''
if cred is not None:
(host, realm, username, password) = cred
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm, host, username, password)
if auth_handler:
opener = build_opener(cookie_processor, auth_handler)
else:
opener = build_opener(cookie_processor)
install_opener(opener)
print("URL %s" % url)
data = urlencode(args)
req = Request(url, data)
f = urlopen(req)
return f
def send_challonge_request(challonge_path, current_user):
data = {} # needed so the Request object is a "POST" request
req = Request(challonge_path, data)
# use basic authentication
user, api_key = current_user.challonge_username, xor_crypt_string(current_user.api_key, decode=True)
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key
)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except URLLibHTTPError as e:
if e.code != 422:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def token_request(uri, post_data):
"""
This function encapsulates the code used to make the request
to the (reddit) OAuth2 endpoint that is shared between
obtaining the first, refreshing and revoking the token.
The parameters are the endpoint URI and dictionary with the post data
to be JSONified and sent with the POST request to the URI.
"""
request_data = urlencode(post_data).encode('utf-8')
token_request = client_http_request(uri, method='POST', data=request_data)
token_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
token_request.add_header('User-Agent', rwh.config['APP_USER_AGENT_SERVER'])
authenticator = HTTPBasicAuthHandler()
authenticator.add_password(realm='reddit', uri=uri,
user=rwh.config['APP_ID'], passwd='')
authenticated_opener = build_opener(authenticator)
request_result = authenticated_opener.open(token_request)
status_code = request_result.getcode()
result_body = None
if request_result:
result_body = request_result.read().decode('utf-8')
if result_body and (len(result_body.strip()) > 0):
result_data = parse_json_string(result_body)
else:
result_data = None
return result_data, status_code
def build_opener(apiurl, user, password, cookie_path, debuglevel=0, capath=None, cafile=None, headers=()):
"""build urllib opener for given name/password
it creates
* HTTPSHandler with proper ssl context
* HTTPCookieProcessor with a link to cookiejar
* HTTPBasicAuthHandler with user/password
* proxyhandler which respects no_proxy variable
"""
handlers = list()
if hasattr(ssl, "SSLContext"):
#allow only sslv3 and tlsv1, but not sslv2
ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_SSLv23)
ctx.options |= ssl.OP_NO_SSLv2
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.set_default_verify_paths()
if cafile or capath:
if ctx.load_verify_locations(capath=capath, cafile=cafile) != -1:
raise Exception("load_verify_locations failed for capath={}, cafile={}".format(capath, cafile))
#TODO: debuglevel
httpshandler = HTTPSHandler(debuglevel=debuglevel, context=ctx, check_hostname=True)
handlers.append(httpshandler)
try:
# TODO is this correct?
cookie_file = os.path.expanduser(cookie_path)
cookiejar = LWPCookieJar(cookie_file)
cookiejar.load(ignore_discard=True)
except (OSError, IOError, AttributeError):
try:
os.open(cookie_file, os.O_WRONLY | os.O_CREAT, mode=0o600)
except:
#TODO: log it
cookiejar = CookieJar()
handlers.append(HTTPCookieProcessor(cookiejar))
authhandler = HTTPBasicAuthHandler(
HTTPPasswordMgrWithDefaultRealm())
authhandler.add_password(None, apiurl, bytes(user, "utf-8"), bytes(password, "ascii"))
handlers.append(authhandler)
# proxy handling
if not proxy_bypass(apiurl):
proxyhandler = ProxyHandler()
else:
proxyhandler = ProxyHandler({})
handlers.append(proxyhandler)
opener = _build_opener(*handlers)
from bslib import __version__
opener.addheaders = [("User-agent", "bslib/{}".format(__version__)), ]
for h in headers:
opener.addheaders(h)
return opener
def create_url_opener(cookie_file_path, domain):
"""Load username and password from .gitcookies and return a URL opener with
an authentication handler."""
# Load authentication credentials
credentials = load_auth_credentials(cookie_file_path)
username, password = credentials[domain]
# Create URL opener with authentication handler
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(domain, domain, username, password)
return build_opener(auth_handler)
def download(self, url: str, user: str, password: str, output:str):
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(None, url, user, password)
req = Request(url, headers = {
'Authorization': self.__build_basic_auth(user, password)
})
with urlopen(req) as remote_file:
with self.fs.open(output, 'wb', None) as local_file:
copyfileobj(remote_file, local_file)
return True
def download(self, url: str, user: str, password: str, output: str):
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(None, url, user, password)
req = Request(
url,
headers={'Authorization': self.__build_basic_auth(user, password)})
with urlopen(req) as remote_file:
with self.fs.open(output, 'wb', None) as local_file:
copyfileobj(remote_file, local_file)
return True
def Auth(User, Pass):
"""
This authentication works for the current 2008-2019 physionet authentication
version. It will NOT work for the NEW version of physionet that is being developed.
"""
url='https://archive.physionet.org/works/MIMICIIIClinicalDatabase/files/'
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm='PhysioNetWorks', uri=url, user=User, passwd=Pass)
opener = build_opener(auth_handler)
try:
r = opener.open(url)
return r.getcode()
except HTTPError as e:
return e.getcode()
def fetch(method, uri, params_prefix=None, **params):
"""Fetch the given uri and return the contents of the response."""
# build the HTTP request
is_json = False
if not params.get('json', None):
params = urlencode(_prepare_params(params, params_prefix))
binary_params = params.encode('ASCII')
url = "https://%s/%s.xml" % (CHALLONGE_API_URL, uri)
req = Request(url, binary_params)
req.get_method = lambda: method
else:
url = "https://%s/%s.json" % (CHALLONGE_API_URL, uri)
payload = params.get('data', None)
req = Request(url,
data=payload,
headers={'Content-Type': 'application/json'})
req.get_method = lambda: method
is_json = True
# use basic authentication
user, api_key = get_credentials()
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm="Application",
uri=req.get_full_url(),
user=user,
passwd=api_key)
opener = build_opener(auth_handler)
try:
response = opener.open(req)
except HTTPError as e:
if e.code != 422:
raise
if is_json:
msg = e.readline()
if 'errors' in msg:
raise ChallongeException(msg)
else:
raise
# wrap up application-level errors
doc = ElementTree.parse(e).getroot()
if doc.tag != "errors":
raise
errors = [e.text for e in doc]
raise ChallongeException(*errors)
return response
def __init__(self,
cachedir='/tmp',
api_host_options={},
urllist=[],
http_debug=False,
cookiejar=None,
offline=False,
enable_cpio=True):
# set up progress bar callback
self.progress_obj = None
if sys.stdout.isatty():
self.progress_obj = create_text_meter(use_pb_fallback=False)
self.cachedir = cachedir
self.urllist = urllist
self.http_debug = http_debug
self.offline = offline
self.cpio = {}
self.enable_cpio = enable_cpio
passmgr = HTTPPasswordMgrWithDefaultRealm()
for host in api_host_options:
passmgr.add_password(None, host, api_host_options[host]['user'],
api_host_options[host]['pass'])
openers = (HTTPBasicAuthHandler(passmgr), )
if cookiejar:
openers += (HTTPCookieProcessor(cookiejar), )
self.gr = OscFileGrabber(progress_obj=self.progress_obj)
def execute(self, method, *args, **kwargs):
header = {
'Content-Type' : 'application/json',
'User-Agent' : 'python-xbmc'
}
# Params are given as a dictionnary
if len(args) == 1:
args=args[0]
params = kwargs
# Use kwargs for param=value style
else:
args = kwargs
params={}
params['jsonrpc']='2.0'
params['id']=self.id
self.id +=1
params['method']=method
params['params']=args
values=json.dumps(params)
# HTTP Authentication
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, self.url, self.username, self.password)
auth_handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(auth_handler)
install_opener(opener)
data = values
req = Request(self.url, data.encode('utf-8'), header)
response = urlopen(req)
the_page = response.read()
if len(the_page) > 0 :
return json.load(StringIO(the_page.decode('utf-8')))
else:
return None # for readability
def basic_authentication_opener_factory(
registry_url: str,
username: str,
password: str,
) -> OpenerDirector:
"""
Return an opener director that authenticates requests using the Basic scheme.
Args:
registry_url (str): The URL of the container registry API.
username (str): A username used for Basic authentication when retrieving an
access token.
password (str): A password used for Basic authentication when
retrieving an access token.
Returns:
OpenerDirector: A director for making HTTP requests. It's main method is `open`.
"""
opener = opener_factory()
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(
None, # type: ignore
registry_url,
username,
password,
)
opener.add_handler(HTTPBasicAuthHandler(password_manager))
return opener
def query(self, query, ts_start, ts_end):
# target = 'summarize({},"{}","avg")'.format(
# query, '99year') @TODO remove if not needed
# build graphite url
args = {
'__auth_token': self.token,
'target': query,
'format': 'json',
'from': str(ts_start),
'until': str(ts_end),
}
url = '{}/render?'.format(self.url)
for k, v in args.items():
url += '{}={}&'.format(quote(k), quote(v))
# Basic auth header
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None,
self.graphite_url,
self.username,
self.password,
)
auth_handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(auth_handler)
install_opener(opener)
result = json.loads(urlopen(url).read().decode('utf-8'))
return result
def urlopen(url, headers=None, data=None, timeout=None):
"""
An URL opener with the User-agent set to gPodder (with version)
"""
username, password = username_password_from_url(url)
if username is not None or password is not None:
url = url_strip_authentication(url)
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None, url, username, password)
handler = HTTPBasicAuthHandler(password_mgr)
opener = build_opener(handler)
else:
opener = build_opener()
if headers is None:
headers = {}
else:
headers = dict(headers)
headers.update({'User-agent': USER_AGENT})
request = Request(url, data=data, headers=headers)
if timeout is None:
return opener.open(request)
else:
return opener.open(request, timeout=timeout)
def graph(request):
"""Proxy Graphite graphs
We don't want to bother the user with authenticating to Graphite.
Instead, here we download the graph using our credentials and pass
it to the user.
"""
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None,
settings.GRAPHITE_URL,
settings.GRAPHITE_USER,
settings.GRAPHITE_PASSWORD,
)
auth_handler = HTTPBasicAuthHandler(password_mgr)
url = '{0}/render?{1}'.format(settings.GRAPHITE_URL,
request.GET.urlencode())
# If the Graphite server fails, we would return proper server error
# to the user instead of failing. This is not really a matter for
# the user as they would get a 500 in any case, but it is a matter for
# the server. We expect any kind of IO error in here, but the socket
# errors are more likely to happen. Graphite has the tendency to return
# empty result with 200 instead of proper error codes.
try:
with build_opener(auth_handler).open(url) as response:
return HttpResponse(response.read(), content_type='image/png')
except IOError as error:
return HttpResponseServerError(str(error))
def _install_opener(self):
self.opener = None
if self.username != "":
self.password_mgr = HTTPPasswordMgrWithDefaultRealm()
self.password_mgr.add_password(None, self.baseurlauth,
self.username, self.password)
self.auth_handler = HTTPBasicAuthHandler(self.password_mgr)
if self.ssl is True:
if self.cafile == "":
self.context = ssl.create_default_context()
self.context.check_hostname = False
self.context.verify_mode = ssl.CERT_NONE
else:
self.context = ssl.create_default_context()
self.context.load_verify_locations(cafile=self.cafile)
self.context.verify_mode = ssl.CERT_REQUIRED
self.https_handler = HTTPSHandler(context=self.context)
if self.username != "":
self.opener = build_opener(self.https_handler,
self.auth_handler)
else:
self.opener = build_opener(self.https_handler)
else:
if self.username != "":
self.opener = build_opener(self.auth_handler)
if self.opener is not None:
self.log.debug("Setting up opener on: {}".format(self.baseurlauth))
install_opener(self.opener)
def download(self, source, dest):
"""
Download an archive file.
:param str source: URL pointing to an archive file.
:param str dest: Local path location to download archive file to.
"""
# propogate all exceptions
# URLError, OSError, etc
proto, netloc, path, params, query, fragment = urlparse(source)
if proto in ('http', 'https'):
auth, barehost = splituser(netloc)
if auth is not None:
source = urlunparse(
(proto, barehost, path, params, query, fragment))
username, password = splitpasswd(auth)
passman = HTTPPasswordMgrWithDefaultRealm()
# Realm is set to None in add_password to force the username and password
# to be used whatever the realm
passman.add_password(None, source, username, password)
authhandler = HTTPBasicAuthHandler(passman)
opener = build_opener(authhandler)
install_opener(opener)
response = urlopen(source)
try:
with open(dest, 'wb') as dest_file:
dest_file.write(response.read())
except Exception as e:
if os.path.isfile(dest):
os.unlink(dest)
raise e
def __open(self, url, headers={}, data=None, baseurl=""):
"""Raw urlopen command"""
if not baseurl:
baseurl = self.baseurl
req = Request("%s%s" % (baseurl, url), headers=headers)
try:
req.data = urlencode(data).encode('utf-8') # Python 3
except:
try:
req.add_data(urlencode(data)) # Python 2
except:
pass
# Proxy support
if self.proxy_url:
if self.proxy_user:
proxy = ProxyHandler({
'https':
'https://%s:%s@%s' %
(self.proxy_user, self.proxy_password, self.proxy_url)
})
auth = HTTPBasicAuthHandler()
opener = build_opener(proxy, auth, HTTPHandler)
else:
handler = ProxyHandler({'https': self.proxy_url})
opener = build_opener(handler)
else:
opener = build_opener()
resp = opener.open(req)
charset = resp.info().get('charset', 'utf-8')
return json.loads(resp.read().decode(charset))
def init_auth(user, password, url):
""" Init authentication """
pass_mgr = HTTPPasswordMgrWithDefaultRealm()
pass_mgr.add_password(None, url, user, password)
auth_handler = HTTPBasicAuthHandler(pass_mgr)
opener = build_opener(auth_handler)
install_opener(opener)
def authenticate(self, username, password):
manager = HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, BASE_URL_AUTHENTICATED, username, password)
handler = HTTPBasicAuthHandler(manager)
self.baseurl = BASE_URL_AUTHENTICATED
self.opener.add_handler(handler)
def foreach(self, server):
"""Helper function to iterate Graphite metrics using foreach_path
We will return an empty element even though foreach_path is not set
for convenience of the caller.
"""
if self.foreach_path:
formatter = AttributeFormatter()
params = formatter.vformat('query=' + self.foreach_path, (),
server)
password_mgr = HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(
None,
settings.GRAPHITE_URL,
settings.GRAPHITE_USER,
settings.GRAPHITE_PASSWORD,
)
auth_handler = HTTPBasicAuthHandler(password_mgr)
url = '{0}/metrics/find?{1}'.format(settings.GRAPHITE_URL, params)
with build_opener(auth_handler).open(url) as response:
return json.loads(response.read().decode())
return [{
'id': '',
'leaf': 0,
'context': {},
'text': '',
'expandable': 0,
'allowChildren': 0,
}]
def __call__(self, *args, **kwargs):
try:
from urllib.request import install_opener, build_opener, ProxyHandler, HTTPBasicAuthHandler
except ImportError:
#from urllib2 import install_opener, build_opener, ProxyHandler, HTTPBasicAuthHandler
pass
r = requests.get('http://127.0.0.1:8000/?types=0&count=20&country=国内')
ip_ports = json.loads(r.text)
print(ip_ports)
import random
randomIndex = random.randint(0, 19)
ip = ip_ports[randomIndex][0]
port = ip_ports[randomIndex][1]
proxies = {
'http': 'http://%s:%s' % (ip, port),
'https': 'http://%s:%s' % (ip, port)
}
proxy_support = ProxyHandler(proxies)
proxy_auth_handler = HTTPBasicAuthHandler()
#proxy_auth_handler.add_password('realm', 'host', 'user', 'pwd')
# proxy_auth_handler = None
opener = build_opener(proxy_support, proxy_auth_handler)
install_opener(opener)
return self.fn(*args, **kwargs)
def __init__(self,
server,
directory="",
username=None,
password=None,
proxy="",
timeout=10,
apipath=None,
useHTTPS=True,
source=__name__):
"""
parameters
* server (string) name of the server the account is located on
* directory (string) if the friendica instance is located in a
subdirectory, specify it here
* apipath (string) alternatively to calculate the API path from
server name and installation directory you
can specify the path here
* username (string) account name => username@servername
* password (string) the password for the account
* proxy (string) this proxy will be used for connections
to the server
* timeout (integer) seconds to wait for the response during
network requests, default is 10 seconds
* useHTTPS (boolean) use HTTPS (true) or not (false) default is
to use HTTPS and will fallback to HTTP if
that does not work
* source (string) this string will be used as source string,
e.g. client name, when publishing things
"""
self.server = server
self.directory = directory
if (apipath == None):
self.apipath = self.server + '/' + directory
if len(directory):
if not (directory[-1] == '/'):
self.apipath = self.apipath + '/'
self.apipath = self.apipath + 'api'
else:
self.apipath = apipath
self.username = username
self.password = password
self.proxy = proxy
self.timeout = timeout
self.useHTTPS = useHTTPS
self.source = source
self.cj = CookieJar()
self.pwd_mgr = HTTPPasswordMgrWithDefaultRealm()
self.pwd_mgr.add_password(None,
self.protocol() + self.apipath,
self.username, self.password)
self.handler = HTTPBasicAuthHandler(self.pwd_mgr)
if not self.proxy:
self.opener = build_opener(self.handler,
HTTPCookieProcessor(self.cj))
else:
self.proxy_handler = ProxyHandler({'http': proxy, 'https': proxy})
self.opener = build_opener(self.proxy_handler, self.handler,
HTTPCookieProcessor(self.cj))
def __init__(self, url, user=None, password=None):
self.user = user
self.password = password
self.url = url
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, url, user, password)
auth_manager = HTTPBasicAuthHandler(password_manager)
self.opener = build_opener(auth_manager)
def __init__(self, url, user=None, password=None):
self.user = user
self.password = password
self.url = url
password_manager = HTTPPasswordMgrWithPriorAuth()
password_manager.add_password(None, url, user, password, is_authenticated=True)
auth_manager = HTTPBasicAuthHandler(password_manager)
self.opener = build_opener(auth_manager)
def __open(self, url, headers={}, data=None, baseurl=""):
"""Raw urlopen command"""
if not baseurl:
baseurl = self.baseurl
self._user_agent()
last_except = Exception
for count in range(self.tries):
try:
req = Request("%s%s" % (baseurl, url), headers=headers)
try:
req.data = urlencode(data).encode('utf-8') # Python 3
except:
try:
req.add_data(urlencode(data)) # Python 2
except:
pass
# Proxy support
if self.proxy_url:
if self.proxy_user:
proxy = ProxyHandler({
'https':
'https://%s:%s@%s' %
(self.proxy_user, self.proxy_password,
self.proxy_url)
})
auth = HTTPBasicAuthHandler()
opener = build_opener(proxy, auth, HTTPHandler)
else:
handler = ProxyHandler({'https': self.proxy_url})
opener = build_opener(handler)
else:
opener = build_opener(
HTTPSHandler(debuglevel=self.debuglevel))
resp = opener.open(req, timeout=5)
charset = resp.info().get('charset', 'utf-8')
break
except (HTTPError, URLError, SSLError, socket.timeout) as e:
import sys
last_except = e
print('# %d Timed out or other error for %s: %s\n' %
(time.time(), type, str(e)),
file=sys.stderr)
if self.debug:
print('# %d Timed out or other error for %s: %s\n' %
(time.time(), type, str(e)))
count += 1
if count != self.tries:
time.sleep(count * self.retry_delay)
else:
print("# %d %d exceptions: %s" %
(time.time(), self.tries, str(last_except)))
raise last_except
return json.loads(resp.read().decode(charset))
def set_authentication(self, uri, login, password):
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None,
uri=uri,
user=login,
passwd=password)
self.http_opener = build_opener(
HTTPBasicAuthHandler(password_manager),
HTTPDigestAuthHandler(password_manager))
def create_opener(aur_server_tag: str) -> OpenerDirector:
server = _aur_server(aur_server_tag)
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None,
uri=server.address,
user=server.user,
passwd=server.password)
handler = HTTPBasicAuthHandler(password_manager)
return build_opener(handler)
def setup_proxy_opener():
# check for http[s]?_proxy user
for scheme in ['http', 'https']:
key = scheme + '_proxy'
if key in os.environ:
proxy = ProxyHandler({scheme: os.environ[key]})
auth = HTTPBasicAuthHandler()
opener = build_opener(proxy, auth, HTTPHandler)
install_opener(opener)
def opener(self):
if self._opener is None:
if self.referer is not None and self.referer._opener is not None:
self._opener = self.referer._opener
else:
o = build_opener()
o.add_handler(HTTPBasicAuthHandler(NetrcHTTPPasswordMgr()))
self._opener = o
return self._opener
def _bget(my_url, user=None, passw=None):
"""
Get remote auth-protected url *that contains a ~file~*
and pass its binary data straight from remote response
(for instance when retrieving fulltext from ISTEX API)
"""
# /!\ attention le password est en clair ici /!\
# print ("REGARD:", user, passw, file=stderr)
no_contents = False
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(
realm = 'Authentification sur api.istex.fr',
uri = 'https://api.istex.fr',
user = user,
passwd = passw)
install_opener(build_opener(auth_handler))
print("GET bin (user:%s)" % user, file=stderr)
# contact
try:
remote_file = urlopen(my_url)
except URLError as url_e:
if url_e.getcode() == 401:
raise AuthWarning("need_auth")
else:
# 404 à gérer *sans quitter* pour les fulltexts en nombre...
no_contents = True
print("api: HTTP ERR no %i (%s) sur '%s'" %
(url_e.getcode(),url_e.msg, my_url), file=stderr)
# pour + de détail
# print ("ERR.info(): \n %s" % url_e.info(),file=stderr)
if no_contents:
return None
else:
# lecture
contents = remote_file.read()
remote_file.close()
return contents
def load_auth(cookie_file_path):
"""Load username and password from .gitcookies and return an
HTTPBasicAuthHandler."""
auth_handler = HTTPBasicAuthHandler()
with open(cookie_file_path, 'r') as cookie_file:
for lineno, line in enumerate(cookie_file, start=1):
if line.startswith('#HttpOnly_'):
line = line[len('#HttpOnly_'):]
if not line or line[0] == '#':
continue
row = line.split('\t')
if len(row) != 7:
continue
domain = row[0]
cookie = row[6]
sep = cookie.find('=')
if sep == -1:
continue
username = cookie[0:sep]
password = cookie[sep + 1:]
auth_handler.add_password(domain, domain, username, password)
return auth_handler
def update(self):
if not self.config['url']:
return None
auth = HTTPBasicAuthHandler()
if self.config['realm'] and self.config['user'] \
and self.config['password']:
auth.add_password(self.config['realm'], self.config['url'],
self.config['user'], self.config['password'])
opener = build_opener(auth)
try:
resp = opener.open(self.config['url'])
except HTTPError:
return False
p = feedparser.parse(resp)
if not p.entries:
return False
if self.config['feed_name']:
author = self.config['feed_name']
else:
author = p.feed.title
for entry in p.entries[::-1]:
if 'link' not in entry or 'title' not in entry \
or self.manager.state.check(self.id, entry.link):
continue
link = entry.link
id = entry.link
title = entry.title
body = None
if 'content' in entry and entry.content:
if entry.content[0].type == 'text/html':
body = recent.markup.html_to_log(entry.content[0].value)
else:
body = entry.content[0].value
self.manager.state.add(self.id, entry.link)
n = Notification(provider=self.name, id=id, title=title,
author=author, link=link, body=body)
self.manager.notify(n)
def open_url(url, config, data=None, handlers=None):
"""Attempts to open a connection to a specified URL.
@param url: URL to attempt to open
@param config: SSL context configuration
@type config: Configuration
@param data: HTTP POST data
@type data: str
@param handlers: list of custom urllib2 handlers to add to the request
@type handlers: iterable
@return: tuple (
returned HTTP status code or 0 if an error occurred
returned message or error description
response object)
"""
debuglevel = 1 if config.debug else 0
# Set up handlers for URL opener.
if config.cookie:
cj = config.cookie
else:
cj = cookiejar_.CookieJar()
# Use a cookie processor that accumulates cookies when redirects occur so
# that an application can redirect for authentication and retain both any
# cookies for the application and the security system (c.f.,
# urllib2.HTTPCookieProcessor which replaces cookies).
cookie_handler = AccumulatingHTTPCookieProcessor(cj)
if not handlers:
handlers = []
handlers.append(cookie_handler)
if config.debug:
http_handler = HTTPHandler_(debuglevel=debuglevel)
https_handler = HTTPSContextHandler(config.ssl_context,
debuglevel=debuglevel)
handlers.extend([http_handler, https_handler])
if config.http_basicauth:
# currently only supports http basic auth
auth_handler = HTTPBasicAuthHandler_(HTTPPasswordMgrWithDefaultRealm_())
auth_handler.add_password(realm=None, uri=url,
user=config.http_basicauth[0],
passwd=config.http_basicauth[1])
handlers.append(auth_handler)
# Explicitly remove proxy handling if the host is one listed in the value of
# the no_proxy environment variable because urllib2 does use proxy settings
# set via http_proxy and https_proxy, but does not take the no_proxy value
# into account.
if not _should_use_proxy(url, config.no_proxy):
handlers.append(ProxyHandler_({}))
log.debug("Not using proxy")
elif config.proxies:
handlers.append(ProxyHandler_(config.proxies))
log.debug("Configuring proxies: %s" % config.proxies)
opener = build_opener(*handlers, ssl_context=config.ssl_context)
headers = config.headers
if headers is None:
headers = {}
request = Request_(url, data, headers)
# Open the URL and check the response.
return_code = 0
return_message = ''
response = None
try:
response = opener.open(request)
return_message = response.msg
return_code = response.code
if log.isEnabledFor(logging.DEBUG):
for index, cookie in enumerate(cj):
log.debug("%s : %s", index, cookie)
except HTTPError_ as exc:
return_code = exc.code
return_message = "Error: %s" % exc.msg
if log.isEnabledFor(logging.DEBUG):
log.debug("%s %s", exc.code, exc.msg)
except Exception as exc:
return_message = "Error: %s" % exc.__str__()
if log.isEnabledFor(logging.DEBUG):
import traceback
log.debug(traceback.format_exc())
return (return_code, return_message, response)
