def __init__(self, cachedir='/tmp', api_host_options={}, urllist=[], http_debug=False, cookiejar=None, offline=False, enable_cpio=True): # set up progress bar callback self.progress_obj = None if sys.stdout.isatty(): self.progress_obj = create_text_meter(use_pb_fallback=False) self.cachedir = cachedir self.urllist = urllist self.http_debug = http_debug self.offline = offline self.cpio = {} self.enable_cpio = enable_cpio passmgr = HTTPPasswordMgrWithDefaultRealm() for host in api_host_options: passmgr.add_password(None, host, api_host_options[host]['user'], api_host_options[host]['pass']) openers = (HTTPBasicAuthHandler(passmgr), ) if cookiejar: openers += (HTTPCookieProcessor(cookiejar), ) self.gr = OscFileGrabber(progress_obj=self.progress_obj)
def opener_for_url_prefix(url_prefix, username=None, password=None, cache_dict=None): if cache_dict is not None: cache_key = (url_prefix, username, password) try: return cache_dict[cache_key] except KeyError: pass if username or password: auth_handler = HTTPBasicAuthHandler() auth_handler.add_password(realm="Open Amiga Game Database", uri="{0}".format(url_prefix), user=username, passwd=password) auth_handler.add_password(realm="OpenRetro", uri="{0}".format(url_prefix), user=username, passwd=password) opener = build_opener(auth_handler) else: opener = build_opener() if cache_dict is not None: cache_key = (url_prefix, username, password) cache_dict[cache_key] = opener return opener
def __open(self, url, headers={}, data=None, baseurl=""): """Raw urlopen command""" if not baseurl: baseurl = self.baseurl req = Request("%s%s" % (baseurl, url), headers=headers) try: req.data = urlencode(data).encode('utf-8') # Python 3 except: try: req.add_data(urlencode(data)) # Python 2 except: pass # Proxy support if self.proxy_url: if self.proxy_user: proxy = ProxyHandler({ 'https': 'https://%s:%s@%s' % (self.proxy_user, self.proxy_password, self.proxy_url) }) auth = HTTPBasicAuthHandler() opener = build_opener(proxy, auth, HTTPHandler) else: handler = ProxyHandler({'https': self.proxy_url}) opener = build_opener(handler) else: opener = build_opener() resp = opener.open(req) charset = resp.info().get('charset', 'utf-8') return json.loads(resp.read().decode(charset))
def basic_authentication_opener_factory( registry_url: str, username: str, password: str, ) -> OpenerDirector: """ Return an opener director that authenticates requests using the Basic scheme. Args: registry_url (str): The URL of the container registry API. username (str): A username used for Basic authentication when retrieving an access token. password (str): A password used for Basic authentication when retrieving an access token. Returns: OpenerDirector: A director for making HTTP requests. It's main method is `open`. """ opener = opener_factory() password_manager = HTTPPasswordMgrWithDefaultRealm() password_manager.add_password( None, # type: ignore registry_url, username, password, ) opener.add_handler(HTTPBasicAuthHandler(password_manager)) return opener
def get_auth_handler(cls, feed_url, username, password, logger=None): """ Create a URL that will perform authentication for the given feed. Arguments: feed_url -- The URL of the feed to retrieve (as a string) username -- The username to use when authenticating password -- The password to use when authenticating """ realm, auth_type = cls.get_realm_and_auth_type(feed_url, username, password, logger) # Make the associated auth handler if auth_type == None: return None elif auth_type.lower() == "basic": auth_handler = HTTPBasicAuthHandler() else: auth_handler = HTTPDigestAuthHandler() # Set the password auth_handler.add_password(realm=realm, uri=feed_url, user=username, passwd=password) return auth_handler
def authenticate(self, username, password): manager = HTTPPasswordMgrWithDefaultRealm() manager.add_password(None, BASE_URL_AUTHENTICATED, username, password) handler = HTTPBasicAuthHandler(manager) self.baseurl = BASE_URL_AUTHENTICATED self.opener.add_handler(handler)
def send_challonge_request(challonge_path, current_user): data = {} # needed so the Request object is a "POST" request req = Request(challonge_path, data) # use basic authentication user, api_key = current_user.challonge_username, xor_crypt_string(current_user.api_key, decode=True) auth_handler = HTTPBasicAuthHandler() auth_handler.add_password( realm="Application", uri=req.get_full_url(), user=user, passwd=api_key ) opener = build_opener(auth_handler) try: response = opener.open(req) except URLLibHTTPError as e: if e.code != 422: raise # wrap up application-level errors doc = ElementTree.parse(e).getroot() if doc.tag != "errors": raise errors = [e.text for e in doc] raise ChallongeException(*errors) return response
def execute(self, method, *args, **kwargs): header = { 'Content-Type' : 'application/json', 'User-Agent' : 'python-xbmc' } # Params are given as a dictionnary if len(args) == 1: args=args[0] params = kwargs # Use kwargs for param=value style else: args = kwargs params={} params['jsonrpc']='2.0' params['id']=self.id self.id +=1 params['method']=method params['params']=args values=json.dumps(params) # HTTP Authentication password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, self.url, self.username, self.password) auth_handler = HTTPBasicAuthHandler(password_mgr) opener = build_opener(auth_handler) install_opener(opener) data = values req = Request(self.url, data.encode('utf-8'), header) response = urlopen(req) the_page = response.read() if len(the_page) > 0 : return json.load(StringIO(the_page.decode('utf-8'))) else: return None # for readability
def _register_agent(self): register_name = self.app.config.get('PSDASH_REGISTER_AS') if not register_name: register_name = socket.gethostname() url_args = { 'name': register_name, 'port': self.app.config.get('PSDASH_PORT', self.DEFAULT_PORT), } register_url = '%s/register?%s' % ( self.app.config['PSDASH_REGISTER_TO'], urllib.urlencode(url_args)) if 'PSDASH_AUTH_USERNAME' in self.app.config and 'PSDASH_AUTH_PASSWORD' in self.app.config: auth_handler = HTTPBasicAuthHandler() auth_handler.add_password( realm='psDash login required', uri=register_url, user=self.app.config['PSDASH_AUTH_USERNAME'], passwd=self.app.config['PSDASH_AUTH_PASSWORD']) opener = build_opener(auth_handler) install_opener(opener) try: urlopen(register_url) except HTTPError as e: logger.error('Failed to register agent to "%s": %s', register_url, e)
def __call__(self, *args, **kwargs): try: from urllib.request import install_opener, build_opener, ProxyHandler, HTTPBasicAuthHandler except ImportError: #from urllib2 import install_opener, build_opener, ProxyHandler, HTTPBasicAuthHandler pass r = requests.get('http://127.0.0.1:8000/?types=0&count=20&country=国内') ip_ports = json.loads(r.text) print(ip_ports) import random randomIndex = random.randint(0, 19) ip = ip_ports[randomIndex][0] port = ip_ports[randomIndex][1] proxies = { 'http': 'http://%s:%s' % (ip, port), 'https': 'http://%s:%s' % (ip, port) } proxy_support = ProxyHandler(proxies) proxy_auth_handler = HTTPBasicAuthHandler() #proxy_auth_handler.add_password('realm', 'host', 'user', 'pwd') # proxy_auth_handler = None opener = build_opener(proxy_support, proxy_auth_handler) install_opener(opener) return self.fn(*args, **kwargs)
def query(self, query, ts_start, ts_end): # target = 'summarize({},"{}","avg")'.format( # query, '99year') @TODO remove if not needed # build graphite url args = { '__auth_token': self.token, 'target': query, 'format': 'json', 'from': str(ts_start), 'until': str(ts_end), } url = '{}/render?'.format(self.url) for k, v in args.items(): url += '{}={}&'.format(quote(k), quote(v)) # Basic auth header password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password( None, self.graphite_url, self.username, self.password, ) auth_handler = HTTPBasicAuthHandler(password_mgr) opener = build_opener(auth_handler) install_opener(opener) result = json.loads(urlopen(url).read().decode('utf-8')) return result
def download(self, source, dest): """ Download an archive file. :param str source: URL pointing to an archive file. :param str dest: Local path location to download archive file to. """ # propogate all exceptions # URLError, OSError, etc proto, netloc, path, params, query, fragment = urlparse(source) if proto in ('http', 'https'): auth, barehost = splituser(netloc) if auth is not None: source = urlunparse( (proto, barehost, path, params, query, fragment)) username, password = splitpasswd(auth) passman = HTTPPasswordMgrWithDefaultRealm() # Realm is set to None in add_password to force the username and password # to be used whatever the realm passman.add_password(None, source, username, password) authhandler = HTTPBasicAuthHandler(passman) opener = build_opener(authhandler) install_opener(opener) response = urlopen(source) try: with open(dest, 'wb') as dest_file: dest_file.write(response.read()) except Exception as e: if os.path.isfile(dest): os.unlink(dest) raise e
def foreach(self, server): """Helper function to iterate Graphite metrics using foreach_path We will return an empty element even though foreach_path is not set for convenience of the caller. """ if self.foreach_path: formatter = AttributeFormatter() params = formatter.vformat('query=' + self.foreach_path, (), server) password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password( None, settings.GRAPHITE_URL, settings.GRAPHITE_USER, settings.GRAPHITE_PASSWORD, ) auth_handler = HTTPBasicAuthHandler(password_mgr) url = '{0}/metrics/find?{1}'.format(settings.GRAPHITE_URL, params) with build_opener(auth_handler).open(url) as response: return json.loads(response.read().decode()) return [{ 'id': '', 'leaf': 0, 'context': {}, 'text': '', 'expandable': 0, 'allowChildren': 0, }]
def graph(request): """Proxy Graphite graphs We don't want to bother the user with authenticating to Graphite. Instead, here we download the graph using our credentials and pass it to the user. """ password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password( None, settings.GRAPHITE_URL, settings.GRAPHITE_USER, settings.GRAPHITE_PASSWORD, ) auth_handler = HTTPBasicAuthHandler(password_mgr) url = '{0}/render?{1}'.format(settings.GRAPHITE_URL, request.GET.urlencode()) # If the Graphite server fails, we would return proper server error # to the user instead of failing. This is not really a matter for # the user as they would get a 500 in any case, but it is a matter for # the server. We expect any kind of IO error in here, but the socket # errors are more likely to happen. Graphite has the tendency to return # empty result with 200 instead of proper error codes. try: with build_opener(auth_handler).open(url) as response: return HttpResponse(response.read(), content_type='image/png') except IOError as error: return HttpResponseServerError(str(error))
def urlopen(url, headers=None, data=None, timeout=None): """ An URL opener with the User-agent set to gPodder (with version) """ username, password = username_password_from_url(url) if username is not None or password is not None: url = url_strip_authentication(url) password_mgr = HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, url, username, password) handler = HTTPBasicAuthHandler(password_mgr) opener = build_opener(handler) else: opener = build_opener() if headers is None: headers = {} else: headers = dict(headers) headers.update({'User-agent': USER_AGENT}) request = Request(url, data=data, headers=headers) if timeout is None: return opener.open(request) else: return opener.open(request, timeout=timeout)
def POST(url, args={}, cred=None): """do http post url is the URL you want args is a dict of cgi args cred is ( host, realm, username, password ) """ auth_handler = None arg_string = '' if cred is not None: (host, realm, username, password) = cred auth_handler = HTTPBasicAuthHandler() auth_handler.add_password(realm, host, username, password) if auth_handler: opener = build_opener(cookie_processor, auth_handler) else: opener = build_opener(cookie_processor) install_opener(opener) print("URL %s" % url) data = urlencode(args) req = Request(url, data) f = urlopen(req) return f
def fetch(method, uri, params_prefix=None, **params): """Fetch the given uri and return the contents of the response.""" params = urlencode(_prepare_params(params, params_prefix)) binary_params = params.encode('ASCII') # build the HTTP request url = "https://%s/%s.xml" % (CHALLONGE_API_URL, uri) req = Request(url, binary_params) req.get_method = lambda: method # use basic authentication user, api_key = get_credentials() auth_handler = HTTPBasicAuthHandler() auth_handler.add_password(realm="Application", uri=req.get_full_url(), user=user, passwd=api_key) opener = build_opener(auth_handler) try: response = opener.open(req) except HTTPError as e: if e.code != 422: raise # wrap up application-level errors doc = ElementTree.parse(e).getroot() if doc.tag != "errors": raise errors = [e.text for e in doc] raise ChallongeException(*errors) return response
def init_auth(user, password, url): """ Init authentication """ pass_mgr = HTTPPasswordMgrWithDefaultRealm() pass_mgr.add_password(None, url, user, password) auth_handler = HTTPBasicAuthHandler(pass_mgr) opener = build_opener(auth_handler) install_opener(opener)
def _install_opener(self): self.opener = None if self.username != "": self.password_mgr = HTTPPasswordMgrWithDefaultRealm() self.password_mgr.add_password(None, self.baseurlauth, self.username, self.password) self.auth_handler = HTTPBasicAuthHandler(self.password_mgr) if self.ssl is True: if self.cafile == "": self.context = ssl.create_default_context() self.context.check_hostname = False self.context.verify_mode = ssl.CERT_NONE else: self.context = ssl.create_default_context() self.context.load_verify_locations(cafile=self.cafile) self.context.verify_mode = ssl.CERT_REQUIRED self.https_handler = HTTPSHandler(context=self.context) if self.username != "": self.opener = build_opener(self.https_handler, self.auth_handler) else: self.opener = build_opener(self.https_handler) else: if self.username != "": self.opener = build_opener(self.auth_handler) if self.opener is not None: self.log.debug("Setting up opener on: {}".format(self.baseurlauth)) install_opener(self.opener)
def __init__(self, server, directory="", username=None, password=None, proxy="", timeout=10, apipath=None, useHTTPS=True, source=__name__): """ parameters * server (string) name of the server the account is located on * directory (string) if the friendica instance is located in a subdirectory, specify it here * apipath (string) alternatively to calculate the API path from server name and installation directory you can specify the path here * username (string) account name => username@servername * password (string) the password for the account * proxy (string) this proxy will be used for connections to the server * timeout (integer) seconds to wait for the response during network requests, default is 10 seconds * useHTTPS (boolean) use HTTPS (true) or not (false) default is to use HTTPS and will fallback to HTTP if that does not work * source (string) this string will be used as source string, e.g. client name, when publishing things """ self.server = server self.directory = directory if (apipath == None): self.apipath = self.server + '/' + directory if len(directory): if not (directory[-1] == '/'): self.apipath = self.apipath + '/' self.apipath = self.apipath + 'api' else: self.apipath = apipath self.username = username self.password = password self.proxy = proxy self.timeout = timeout self.useHTTPS = useHTTPS self.source = source self.cj = CookieJar() self.pwd_mgr = HTTPPasswordMgrWithDefaultRealm() self.pwd_mgr.add_password(None, self.protocol() + self.apipath, self.username, self.password) self.handler = HTTPBasicAuthHandler(self.pwd_mgr) if not self.proxy: self.opener = build_opener(self.handler, HTTPCookieProcessor(self.cj)) else: self.proxy_handler = ProxyHandler({'http': proxy, 'https': proxy}) self.opener = build_opener(self.proxy_handler, self.handler, HTTPCookieProcessor(self.cj))
def __init__(self, url, user=None, password=None): self.user = user self.password = password self.url = url password_manager = HTTPPasswordMgrWithPriorAuth() password_manager.add_password(None, url, user, password, is_authenticated=True) auth_manager = HTTPBasicAuthHandler(password_manager) self.opener = build_opener(auth_manager)
def __init__(self, url, user=None, password=None): self.user = user self.password = password self.url = url password_manager = HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, url, user, password) auth_manager = HTTPBasicAuthHandler(password_manager) self.opener = build_opener(auth_manager)
def __open(self, url, headers={}, data=None, baseurl=""): """Raw urlopen command""" if not baseurl: baseurl = self.baseurl self._user_agent() last_except = Exception for count in range(self.tries): try: req = Request("%s%s" % (baseurl, url), headers=headers) try: req.data = urlencode(data).encode('utf-8') # Python 3 except: try: req.add_data(urlencode(data)) # Python 2 except: pass # Proxy support if self.proxy_url: if self.proxy_user: proxy = ProxyHandler({ 'https': 'https://%s:%s@%s' % (self.proxy_user, self.proxy_password, self.proxy_url) }) auth = HTTPBasicAuthHandler() opener = build_opener(proxy, auth, HTTPHandler) else: handler = ProxyHandler({'https': self.proxy_url}) opener = build_opener(handler) else: opener = build_opener( HTTPSHandler(debuglevel=self.debuglevel)) resp = opener.open(req, timeout=5) charset = resp.info().get('charset', 'utf-8') break except (HTTPError, URLError, SSLError, socket.timeout) as e: import sys last_except = e print('# %d Timed out or other error for %s: %s\n' % (time.time(), type, str(e)), file=sys.stderr) if self.debug: print('# %d Timed out or other error for %s: %s\n' % (time.time(), type, str(e))) count += 1 if count != self.tries: time.sleep(count * self.retry_delay) else: print("# %d %d exceptions: %s" % (time.time(), self.tries, str(last_except))) raise last_except return json.loads(resp.read().decode(charset))
def setup_proxy_opener(): # check for http[s]?_proxy user for scheme in ['http', 'https']: key = scheme + '_proxy' if key in os.environ: proxy = ProxyHandler({scheme: os.environ[key]}) auth = HTTPBasicAuthHandler() opener = build_opener(proxy, auth, HTTPHandler) install_opener(opener)
def opener(self): if self._opener is None: if self.referer is not None and self.referer._opener is not None: self._opener = self.referer._opener else: o = build_opener() o.add_handler(HTTPBasicAuthHandler(NetrcHTTPPasswordMgr())) self._opener = o return self._opener
def create_opener(aur_server_tag: str) -> OpenerDirector: server = _aur_server(aur_server_tag) password_manager = HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(realm=None, uri=server.address, user=server.user, passwd=server.password) handler = HTTPBasicAuthHandler(password_manager) return build_opener(handler)
def set_authentication(self, uri, login, password): password_manager = HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(realm=None, uri=uri, user=login, passwd=password) self.http_opener = build_opener( HTTPBasicAuthHandler(password_manager), HTTPDigestAuthHandler(password_manager))
def get_web_page(url, username=None, password=None, login_url=None): """Get url page possible with username and password. """ if login_url: # Login via a form cookies = HTTPCookieProcessor() opener = build_opener(cookies) install_opener(opener) opener.open(login_url) try: token = [ x.value for x in cookies.cookiejar if x.name == 'csrftoken' ][0] except IndexError: return False, "no csrftoken" params = dict( username=username, password=password, this_is_the_login_form=True, csrfmiddlewaretoken=token, ) encoded_params = urlencode(params) with contextlib.closing(opener.open(login_url, encoded_params)) as f: f.read() elif username is not None: # Login using basic auth # Create password manager passman = HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, url, username, password) # create the handler authhandler = HTTPBasicAuthHandler(passman) opener = build_opener(authhandler) install_opener(opener) try: pagehandle = urlopen(url) except HTTPError as e: msg = ('The server couldn\'t fulfill the request. ' 'Error code: %s' % e.status_code) e.args = (msg, ) raise except URLError as e: msg = f'Could not open URL "{url}": {e}' e.args = (msg, ) raise else: page = pagehandle.read() return page
def u2handlers(self): handlers = [] handlers.append(ProxyHandler(self.proxy)) handlers.append( HTTPBasicAuthHandler()) # python ssl Context support - PEP 0466 if hasattr(ssl, '_create_unverified_context'): ssl_context = ssl._create_unverified_context() handlers.append(HTTPSHandler(context=ssl_context)) else: handlers.append(HTTPSHandler()) return handlers
def __init__(self, *, username='******', password='******', top_level_url="https://urs.earthdata.nasa.gov/"): auth_manager = HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, top_level_url, username, password) handler = HTTPBasicAuthHandler(auth_manager) self.opener = urllib.request.build_opener( handler, HTTPCookieProcessor(CookieJar()))