def _read_from_url(url): """Reads data from *url* with an HTTP *GET*. This function supports fetching from resources which use basic HTTP auth as laid out by RFC1738 § 3.1. See § 5 for grammar definitions for URLs. .. seealso: https://www.ietf.org/rfc/rfc1738.txt :param url: URL of an HTTP resource :type url: ``str`` :return: data read from resource described by *url* :rtype: ``file``-like object """ url, username, password = _strip_basic_auth(url) if username is not None and password is not None: # case: url contains basic auth creds password_mgr = request.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, url, username, password) handler = request.HTTPBasicAuthHandler(password_mgr) opener = request.build_opener(default_handlers + [handler]) else: opener = default_opener return opener.open(url)
def __init__(self, config=None): self.tool_sheds = odict() self.tool_sheds_auth = odict() if config: # Parse tool_sheds_conf.xml tree, error_message = xml_util.parse_xml(config) if tree is None: log.warning("Unable to load references to tool sheds defined in file %s" % str(config)) else: root = tree.getroot() log.debug('Loading references to tool sheds from %s' % config) for elem in root.findall('tool_shed'): try: name = elem.get('name', None) url = elem.get('url', None) username = elem.get('user', None) password = elem.get('pass', None) if name and url: self.tool_sheds[name] = url self.tool_sheds_auth[name] = None log.debug('Loaded reference to tool shed: %s' % name) if name and url and username and password: pass_mgr = urlrequest.HTTPPasswordMgrWithDefaultRealm() pass_mgr.add_password(None, url, username, password) self.tool_sheds_auth[name] = pass_mgr except Exception as e: log.warning('Error loading reference to tool shed "%s", problem: %s' % (name, str(e)))
def opener(self, purl, puser=None, ppass=None, pscheme=('http', 'https')): if REDIRECT: h1, h2 = REDIRECT.BindableHTTPHandler, REDIRECT.BindableHTTPSHandler else: h1, h2 = request.HTTPHandler, request.HTTPSHandler handlers = [h1(), h2(), request.HTTPCookieProcessor(self.cookies)] if self.pref != 'noproxy' and purl and len(purl) > 1: #if not noproxy and a proxy url is provided (and its not the placeholder url ie noproxy=':') add a handler handlers += [ request.ProxyHandler({ps: purl for ps in pscheme}), ] #handlers += [request.ProxyHandler({ps:purl}) for ps in pscheme] if puser and ppass: #if proxy user/pass provided and a proxy auth handler pm = request.HTTPPasswordMgrWithDefaultRealm() pm.add_password(None, purl, puser, ppass) handlers += [ request.ProxyBasicAuthHandler(pm), ] return request.build_opener(*handlers)
def pytest_unconfigure(config): """ Called after all tests are completed. """ global enabled, submit_url, commit, branch, environment, username, password if not enabled: return executable = "{}-{}-{}".format(platform.python_implementation(), platform.python_version(), platform.system()) data = [ x.to_codespeed_dict(commit=commit, branch=branch, environment=environment, executable=executable) for x in function_profile_list ] try: json_submit_url = submit_url + 'result/add/json/' if username: password_mgr = urllib_request.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, json_submit_url, username, password) handler = urllib_request.HTTPBasicAuthHandler(password_mgr) opener = urllib_request.build_opener(handler) else: opener = urllib_request.build_opener() # use the opener to fetch a URL f = opener.open(json_submit_url, urlencode({ 'json': json.dumps(data) }).encode('UTF-8')) response = f.read() except HTTPError as e: print('Error while connecting to Codespeed:') print('Exception: {}'.format(str(e))) fd, name = tempfile.mkstemp(suffix='.html') os.close(fd) with open(name, 'wb') as f: f.write(e.read()) print('HTTP Response written to {}'.format(name)) raise e if not response.startswith( 'All result data saved successfully'.encode('UTF-8')): print("Unexpected response while connecting to Codespeed:") raise ValueError( 'Unexpected response from Codespeed server: {}'.format(response)) else: print("{} test benchmarks sumbitted.".format( len(function_profile_list)))
def download_tile(tile, url, pid, version, username, password): grass.debug("Download tile: %s" % tile, debug=1) local_tile = "NASADEM_HGT_" + str(tile) + ".zip" urllib2.urlcleanup() remote_tile = str(url) + "/" + version + "/2000.02.11/" + local_tile goturl = 1 try: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password( None, "https://urs.earthdata.nasa.gov", username, password ) cookie_jar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar), ) urllib2.install_opener(opener) request = urllib2.Request(remote_tile) response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) except: goturl = 0 pass return goturl
def download_tile(tile, url, pid, srtmv3, one, username, password): grass.debug("Download tile: %s" % tile, debug = 1) output = tile + '.r.in.srtm.tmp.' + str(pid) if srtmv3: if one: local_tile = str(tile) + '.SRTMGL1.hgt.zip' else: local_tile = str(tile) + '.SRTMGL3.hgt.zip' else: local_tile = str(tile) + '.hgt.zip' urllib2.urlcleanup() if srtmv3: remote_tile = str(url) + local_tile goturl = 1 try: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, "https://urs.earthdata.nasa.gov", username, password) cookie_jar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), #urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see #urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar)) urllib2.install_opener(opener) request = urllib2.Request(remote_tile) response = urllib2.urlopen(request) fo = open(local_tile, 'w+b') fo.write(response.read()) fo.close time.sleep(0.5) except: goturl = 0 pass return goturl # SRTM subdirs: Africa, Australia, Eurasia, Islands, North_America, South_America for srtmdir in ('Africa', 'Australia', 'Eurasia', 'Islands', 'North_America', 'South_America'): remote_tile = str(url) + str(srtmdir) + '/' + local_tile goturl = 1 try: response = urllib2.urlopen(request) fo = open(local_tile, 'w+b') fo.write(response.read()) fo.close time.sleep(0.5) # does not work: #urllib.urlretrieve(remote_tile, local_tile, data = None) except: goturl = 0 pass if goturl == 1: return 1 return 0
def _open_url(self, url): """ Open an URL 'url' and return the file-like object of the opened URL. """ def _print_warning(timeout): """ This is a small helper function for printing a warning if we cannot open the URL for some time. """ _log.warning("failed to open the URL with %d sec timeout, is the " "proxy configured correctly? Keep trying ..." % timeout) import socket from six.moves import http_client as httplib from six.moves.urllib import request as urllib from six.moves.urllib.error import URLError parsed_url = urlparse.urlparse(url) if parsed_url.scheme == "ssh": # Unfortunately, urllib2 does not handle "ssh://" URLs self._open_url_ssh(parsed_url) return username = parsed_url.username password = parsed_url.password if username and password: # Unfortunately, in order to handle URLs which contain user name # and password (e.g., http://user:[email protected]), we need to # do few extra things. new_url = list(parsed_url) if parsed_url.port: new_url[1] = "%s:%s" % (parsed_url.hostname, parsed_url.port) else: new_url[1] = parsed_url.hostname url = urlparse.urlunparse(new_url) # Build an URL opener which will do the authentication password_manager = urllib.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, url, username, password) auth_handler = urllib.HTTPBasicAuthHandler(password_manager) opener = urllib.build_opener(auth_handler) else: opener = urllib.build_opener() opener.addheaders = [('User-Agent', 'Mozilla/5.0')] urllib.install_opener(opener) # Open the URL. First try with a short timeout, and print a message # which should supposedly give the a clue that something may be going # wrong. # The overall purpose of this is to improve user experience. For # example, if one tries to open a file but did not setup the proxy # environment variables propely, there will be a very long delay before # the failure message. And it is much nicer to pre-warn the user early # about something possibly being wrong. for timeout in (10, None): try: f_obj = opener.open(url, timeout=timeout) # Handling the timeout case in Python 2.7 except socket.timeout as err: if timeout is not None: _print_warning(timeout) else: raise Error("cannot open URL '%s': %s" % (url, err)) except URLError as err: # Handling the timeout case in Python 2.6 if timeout is not None and \ isinstance(err.reason, socket.timeout): _print_warning(timeout) else: raise Error("cannot open URL '%s': %s" % (url, err)) except (IOError, ValueError, httplib.InvalidURL) as err: raise Error("cannot open URL '%s': %s" % (url, err)) except httplib.BadStatusLine: raise Error("cannot open URL '%s': server responds with an " "HTTP status code that we don't understand" % url) self.is_url = True self._f_objs.append(f_obj)
def get_handlers(self): handlers = [] if self._verify_cert == False: ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE handler = request.HTTPSHandler(context=ctx) handlers.append(handler) from urllib.request import HTTPRedirectHandler redirect_handler = HTTPRedirectHandler() redirect_handler.max_redirections = 30 redirect_handler.max_repeats = 30 handlers.append(redirect_handler) if self._username and self._password: passman = request.HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, self._parsed_org_url, self._username, self._password) handlers.append(request.HTTPBasicAuthHandler(passman)) passman = request.HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, self._parsed_org_url, self._username, self._password) handlers.append(request.HTTPDigestAuthHandler(passman)) if os.name == 'nt': try: from arcgis._impl.common._iwa import NtlmSspiAuthHandler, KerberosSspiAuthHandler auth_krb = KerberosSspiAuthHandler() handlers.append(auth_krb) try: auth_NTLM = NtlmSspiAuthHandler() handlers.append(auth_NTLM) except: pass except Error as err: _log.error( "winkerberos packages is required for IWA authentication (NTLM and Kerberos)." ) _log.error( "Please install it:\n\tconda install winkerberos") _log.error(str(err)) else: _log.error( 'The GIS uses Integrated Windows Authentication which is currently only supported on the Windows platform' ) if self._auth == "PKI" or \ (self.cert_file is not None and self.key_file is not None): handlers.append( HTTPSClientAuthHandler(self.key_file, self.cert_file)) elif self._portal_connection and \ self._portal_connection.cert_file is not None and \ self._portal_connection.key_file is not None: handlers.append( HTTPSClientAuthHandler(self._portal_connection.key_file, self._portal_connection.cert_file)) cj = cookiejar.CookieJar() if self.proxy_host: # Simple Proxy Support from urllib.request import ProxyHandler if self.proxy_port is None: self.proxy_port = 80 proxies = { "http": "http://%s:%s" % (self.proxy_host, self.proxy_port), "https": "https://%s:%s" % (self.proxy_host, self.proxy_port) } proxy_support = ProxyHandler(proxies) handlers.append(proxy_support) handlers.append(request.HTTPCookieProcessor(cj)) return handlers
def open_url(url, data=None, headers=None, method=None, use_proxy=True, force=False, last_mod_time=None, timeout=10, validate_certs=True, url_username=None, url_password=None, http_agent=None, force_basic_auth=False, follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None): ''' Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3) Does not require the module environment ''' handlers = [] ssl_handler = maybe_add_ssl_handler(url, validate_certs) if ssl_handler: handlers.append(ssl_handler) parsed = generic_urlparse(urlparse(url)) if parsed.scheme != 'ftp': username = url_username if headers is None: headers = {} if username: password = url_password netloc = parsed.netloc elif '@' in parsed.netloc: credentials, netloc = parsed.netloc.split('@', 1) if ':' in credentials: username, password = credentials.split(':', 1) else: username = credentials password = '' parsed_list = parsed.as_list() parsed_list[1] = netloc # reconstruct url without credentials url = urlunparse(parsed_list) if username and not force_basic_auth: passman = urllib_request.HTTPPasswordMgrWithDefaultRealm() # this creates a password manager passman.add_password(None, netloc, username, password) # because we have put None at the start it will always # use this username/password combination for urls # for which `theurl` is a super-url authhandler = urllib_request.HTTPBasicAuthHandler(passman) digest_authhandler = urllib_request.HTTPDigestAuthHandler(passman) # create the AuthHandler handlers.append(authhandler) handlers.append(digest_authhandler) elif username and force_basic_auth: headers["Authorization"] = basic_auth_header(username, password) else: try: rc = netrc.netrc(os.environ.get('NETRC')) login = rc.authenticators(parsed.hostname) except IOError: login = None if login: username, _, password = login if username and password: headers["Authorization"] = basic_auth_header( username, password) if not use_proxy: proxyhandler = urllib_request.ProxyHandler({}) handlers.append(proxyhandler) if HAS_SSLCONTEXT and not validate_certs: # In 2.7.9, the default context validates certificates context = SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 context.options |= ssl.OP_NO_SSLv3 context.verify_mode = ssl.CERT_NONE context.check_hostname = False handlers.append( HTTPSClientAuthHandler(client_cert=client_cert, client_key=client_key, context=context)) elif client_cert: handlers.append( HTTPSClientAuthHandler(client_cert=client_cert, client_key=client_key)) # pre-2.6 versions of python cannot use the custom https # handler, since the socket class is lacking create_connection. # Some python builds lack HTTPS support. if hasattr(socket, 'create_connection') and CustomHTTPSHandler: handlers.append(CustomHTTPSHandler) handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs)) # add some nicer cookie handling if cookies is not None: handlers.append(urllib_request.HTTPCookieProcessor(cookies)) opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener) data = to_bytes(data, nonstring='passthru') if method: if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'PATCH'): raise ConnectionError('invalid HTTP request method; %s' % method.upper()) request = RequestWithMethod(url, method.upper(), data) else: request = urllib_request.Request(url, data) # add the custom agent header, to help prevent issues # with sites that block the default urllib agent string if http_agent: request.add_header('User-agent', http_agent) # Cache control # Either we directly force a cache refresh if force: request.add_header('cache-control', 'no-cache') # or we do it if the original is more recent than our copy elif last_mod_time: tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000') request.add_header('If-Modified-Since', tstamp) # user defined headers now, which may override things we've set above if headers: if not isinstance(headers, dict): raise ValueError("headers provided to fetch_url() must be a dict") for header in headers: request.add_header(header, headers[header]) urlopen_args = [request, None] if sys.version_info >= (2, 6, 0): # urlopen in python prior to 2.6.0 did not # have a timeout parameter urlopen_args.append(timeout) r = urllib_request.urlopen(*urlopen_args) return r