def _post_data(self, path, data): norm_path = posixpath.join(CONF.metadata_base_url, path) LOG.debug('Posting metadata to: %s', norm_path) req = request.Request(norm_path, data=data) self._get_response(req) return True
def delete(self, endpoint): req = request.Request(self.url + endpoint) req.get_method = lambda: 'DELETE' return self._open(req)
def _get_data(self, path): norm_path = posixpath.join(CONF.metadata_base_url, path) LOG.debug('Getting metadata from: %s', norm_path) req = request.Request(norm_path) response = self._get_response(req) return response.read()
def get(self, path, params=None, ssl=False, compress=True, try_json=True, is_retry=False, use_ordered_dict=False, out_folder=None, file_name=None, force_bytes=False, **kwargs): """ Returns result of an HTTP GET. Handles token timeout and all SSL mode.""" path = quote(path, ':/%') url = path if url.lower().find("https://") > -1 or\ url.lower().find("http://") > -1: url = path elif len(url) == 0: url = self.baseurl elif (len(url) > 0 and url[0] == '/' ) == False and \ self.baseurl.endswith('/') == False: url = "/{path}".format(path=url) if not url.startswith('http://') and \ not url.startswith('https://'): url = self.baseurl + url if ssl or self.all_ssl: url = url.replace('http://', 'https://') # Add the token if logged in if params is None: params = {} if try_json: params['f'] = 'json' if self.is_logged_in: params['token'] = self.token if len(params.keys()) > 0: params = {k: jsonize_dict(v) for k, v in params.items()} url = "{url}?{params}".format(url=url, params=urlencode(params)) #url = self._url_add_token(url, self.token) _log.debug('REQUEST (get): ' + url) try: # Send the request and read the response headers = [('User-Agent', self._useragent)] if self._referer: # and \ #self._auth.lower() != 'pki': headers.append(('Referer', self._referer)) if compress: headers.append(('Accept-encoding', 'gzip')) if self._handlers is None: self._handlers = self.get_handlers() opener = request.build_opener(*self._handlers) opener.addheaders = headers req = request.Request(url, headers={i[0]: i[1] for i in headers}) resp = opener.open(req) resp_data = self._process_response(resp, out_folder=out_folder, file_name=file_name) # if the response is a file saved to disk, return it. if (len(resp_data) < 32767) and os.path.isfile(resp_data): if force_bytes: return open(resp_data, 'rb').read() return resp_data # If we're not trying to parse to JSON, return response as is if not try_json: return resp_data try: if use_ordered_dict: resp_json = json.loads(resp_data, object_pairs_hook=OrderedDict) else: resp_json = json.loads(resp_data) # Check for errors, and handle the case where the token timed # out during use (and simply needs to be re-generated) try: if resp_json.get('error', None): errorcode = resp_json['error']['code'] if errorcode == 498 and not is_retry: _log.info('Token expired during get request, ' \ + 'fetching a new token and retrying') newtoken = self.relogin() newpath = self._url_add_token(path, newtoken) return self.get(path=newpath, params=params, ssl=ssl, compress=compress, try_json=try_json, is_retry=True) elif errorcode == 498: raise RuntimeError('Invalid token') elif errorcode == 403: message = resp_json['error'][ 'message'] if 'message' in resp_json[ 'error'] else '' if message == "SSL Required": return self.get(path=path, params=params, ssl=True, compress=compress, try_json=try_json, is_retry=True) self._handle_json_error(resp_json['error']) return None except AttributeError: # Top-level JSON object isnt a dict, so can't have an error pass # If the JSON parsed correctly and there are no errors, # return the JSON #if 'status' in resp_json: # FOR DEMO TODO REMOVE ME # return resp_json['status'] == 'success' return resp_json # If we couldnt parse the response to JSON, return it as is except ValueError: return resp # If we got an HTTPError when making the request check to see if it's # related to token timeout, in which case, regenerate a token except HTTPError as e: if e.code == 498 and not is_retry: _log.info('Token expired during get request, fetching a new ' \ + 'token and retrying') self.logout() newtoken = self.relogin() newpath = self._url_add_token(path, newtoken) return self.get(newpath, ssl, try_json, is_retry=True) elif e.code == 498: raise RuntimeError('Invalid token') else: raise e
def post(self, endpoint, data=None, content_type="application/json"): if not data: data = {} req = request.Request(self.url + endpoint, data=json.dumps(data)) req.add_header('Content-Type', content_type) return self._open(req)
def download_scripts(proxies=None, install_dir=None): print("Downloading scripts. It might take a while.") # location in which to download stuff: if install_dir is None: import visdom install_dir = os.path.dirname(visdom.__file__) # all files that need to be downloaded: b = 'https://unpkg.com/' bb = '%[email protected]/dist/' % b ext_files = { '%[email protected]/dist/jquery.min.js' % b: 'jquery.min.js', '%[email protected]/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js', '%[email protected]/css/styles.css' % b: 'react-resizable-styles.css', '%[email protected]/css/styles.css' % b: 'react-grid-layout-styles.css', '%[email protected]/dist/react-modal.min.js' % b: 'react-modal.min.js', '%[email protected]/dist/react.min.js' % b: 'react-react.min.js', '%[email protected]/dist/react-dom.min.js' % b: 'react-dom.min.js', '%[email protected]' % b: 'classnames', '%[email protected]' % b: 'layout_bin_packer', 'https://raw.githubusercontent.com/STRML/react-grid-layout/0.14.0/dist/' + 'react-grid-layout.min.js': 'react-grid-layout.min.js', 'https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_SVG': 'mathjax-MathJax.js', # here is another url in case the cdn breaks down again. # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js 'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js', '%scss/bootstrap.min.css' % bb: 'bootstrap.min.css', '%sfonts/glyphicons-halflings-regular.eot' % bb: 'glyphicons-halflings-regular.eot', '%sfonts/glyphicons-halflings-regular.woff2' % bb: 'glyphicons-halflings-regular.woff2', '%sfonts/glyphicons-halflings-regular.woff' % bb: 'glyphicons-halflings-regular.woff', '%sfonts/glyphicons-halflings-regular.ttf' % bb: 'glyphicons-halflings-regular.ttf', '%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: 'glyphicons-halflings-regular.svg#glyphicons_halflingsregular', } # make sure all relevant folders exist: dir_list = [ '%s' % install_dir, '%s/static' % install_dir, '%s/static/js' % install_dir, '%s/static/css' % install_dir, '%s/static/fonts' % install_dir, ] for directory in dir_list: if not os.path.exists(directory): os.makedirs(directory) # set up proxy handler: from six.moves.urllib import request from six.moves.urllib.error import HTTPError, URLError handler = request.ProxyHandler(proxies) if proxies is not None \ else request.BaseHandler() opener = request.build_opener(handler) request.install_opener(opener) # download files one-by-one: for (key, val) in ext_files.items(): # set subdirectory: sub_dir = 'fonts' if '.js' in key: sub_dir = 'js' if '.css' in key: sub_dir = 'css' # download file: filename = '%s/static/%s/%s' % (install_dir, sub_dir, val) if not os.path.exists(filename): req = request.Request(key, headers={'User-Agent': 'Chrome/30.0.0.0'}) try: data = opener.open(req).read() with open(filename, 'wb') as fwrite: fwrite.write(data) except HTTPError as exc: logging.error('Error {} while downloading {}'.format( exc.code, key)) except URLError as exc: logging.error('Error {} while downloading {}'.format( exc.reason, key))
def download_tile(tile, url, pid, srtmv3, one, username, password): grass.debug("Download tile: %s" % tile, debug=1) output = tile + ".r.in.srtm.tmp." + str(pid) if srtmv3: if one: local_tile = str(tile) + ".SRTMGL1.hgt.zip" else: local_tile = str(tile) + ".SRTMGL3.hgt.zip" else: local_tile = str(tile) + ".hgt.zip" urllib2.urlcleanup() if srtmv3: remote_tile = str(url) + local_tile goturl = 1 try: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password( None, "https://urs.earthdata.nasa.gov", username, password ) cookie_jar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar), ) urllib2.install_opener(opener) request = urllib2.Request(remote_tile) response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) except: goturl = 0 pass return goturl # SRTM subdirs: Africa, Australia, Eurasia, Islands, North_America, South_America for srtmdir in ( "Africa", "Australia", "Eurasia", "Islands", "North_America", "South_America", ): remote_tile = str(url) + str(srtmdir) + "/" + local_tile goturl = 1 try: response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) # does not work: # urllib.urlretrieve(remote_tile, local_tile, data = None) except: goturl = 0 pass if goturl == 1: return 1 return 0
def _request_compute_metadata(path): req = request.Request('%s/computeMetadata/v1/%s' % (_GCE_METADATA_ENDPOINT, path), headers={'Metadata-Flavor': 'Google'}) resp = request.urlopen(req) return compat.as_bytes(resp.read())
def _post(self, url, param_dict={}, files={}, securityHandler=None, additional_headers={}, custom_handlers=[], proxy_url=None, proxy_port=80, compress=True, out_folder=None, file_name=None): """ Performs a POST operation on a URL. Inputs: param_dict - key/value pair of values ex: {"foo": "bar"} files - key/value pair of file objects where the key is the input name and the value is the file path ex: {"file": r"c:\temp\myfile.zip"} securityHandler - object that handles the token or other site security. It must inherit from the base security class. ex: arcrest.AGOLSecurityHandler("SomeUsername", "SOMEPASSWORD") additional_headers - are additional key/value headers that a user wants to pass during the operation. ex: {"accept-encoding": "gzip"} custom_handlers - this is additional web operation handlers as a list of objects. Ex: [CustomAuthHandler] proxy_url - url of the proxy proxy_port - default 80, port number of the proxy compress - default true, determines if gzip should be used of not for the web operation. out_folder - if the URL requested returns a file, this will be the disk save location file_name - if the operation returns a file and the file name is not given in the header or a user wishes to override the return saved file name, provide value here. Output: returns dictionary or string depending on web operation. """ self._last_method = "POST" headers = {} opener = None return_value = None handlers = [RedirectHandler()] param_dict, handler, cj = self._processHandler(securityHandler, param_dict) if handler is not None: handlers.append(handler) if cj is not None: handlers.append(request.HTTPCookieProcessor(cj)) if isinstance(custom_handlers, list) and \ len(custom_handlers) > 0: for h in custom_handlers: handlers.append(h) if compress: headers['Accept-Encoding'] = 'gzip' else: headers['Accept-Encoding'] = '' for k, v in additional_headers.items(): headers[k] = v del k, v opener = request.build_opener(*handlers) request.install_opener(opener) opener.addheaders = [(k, v) for k, v in headers.items()] if len(files) == 0: data = urlencode(param_dict) if self.PY3: data = data.encode('ascii') opener.data = data resp = opener.open(url, data=data) else: mpf = MultiPartForm(param_dict=param_dict, files=files) req = request.Request(url) body = mpf.make_result req.add_header('User-agent', self.useragent) req.add_header('Content-type', mpf.get_content_type()) req.add_header('Content-length', len(body)) req.data = body resp = request.urlopen(req) self._last_code = resp.getcode() self._last_url = resp.geturl() return_value = self._process_response(resp=resp, out_folder=out_folder) if isinstance(return_value, dict): if "error" in return_value and \ 'message' in return_value['error']: if return_value['error']['message'].lower( ) == 'request not made over ssl': if url.startswith('http://'): url = url.replace('http://', 'https://') return self._post(url, param_dict, files, securityHandler, additional_headers, custom_handlers, proxy_url, proxy_port, compress, out_folder, file_name) return return_value else: return return_value return return_value
def download_scripts(proxies=None, install_dir=None): import visdom print("Downloading scripts. It might take a while.") # location in which to download stuff: if install_dir is None: install_dir = os.path.dirname(visdom.__file__) # all files that need to be downloaded: b = 'https://unpkg.com/' bb = '%[email protected]/dist/' % b ext_files = { # - js '%[email protected]/dist/jquery.min.js' % b: 'jquery.min.js', '%[email protected]/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js', '%[email protected]/umd/react.production.min.js' % b: 'react-react.min.js', '%[email protected]/umd/react-dom.production.min.js' % b: 'react-dom.min.js', # noqa '%[email protected]/dist/react-modal.min.js' % b: 'react-modal.min.js', # noqa 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_SVG': # noqa 'mathjax-MathJax.js', # here is another url in case the cdn breaks down again. # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js 'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js', # Stanford Javascript Crypto Library for Password Hashing '%[email protected]/sjcl.js' % b: 'sjcl.js', # - css '%[email protected]/css/styles.css' % b: 'react-resizable-styles.css', # noqa '%[email protected]/css/styles.css' % b: 'react-grid-layout-styles.css', # noqa '%scss/bootstrap.min.css' % bb: 'bootstrap.min.css', # - fonts '%[email protected]' % b: 'classnames', '%[email protected]' % b: 'layout_bin_packer', '%sfonts/glyphicons-halflings-regular.eot' % bb: 'glyphicons-halflings-regular.eot', '%sfonts/glyphicons-halflings-regular.woff2' % bb: 'glyphicons-halflings-regular.woff2', '%sfonts/glyphicons-halflings-regular.woff' % bb: 'glyphicons-halflings-regular.woff', '%sfonts/glyphicons-halflings-regular.ttf' % bb: 'glyphicons-halflings-regular.ttf', '%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: # noqa 'glyphicons-halflings-regular.svg#glyphicons_halflingsregular', } # make sure all relevant folders exist: dir_list = [ '%s' % install_dir, '%s/static' % install_dir, '%s/static/js' % install_dir, '%s/static/css' % install_dir, '%s/static/fonts' % install_dir, ] for directory in dir_list: if not os.path.exists(directory): os.makedirs(directory) # set up proxy handler: from six.moves.urllib import request from six.moves.urllib.error import HTTPError, URLError handler = request.ProxyHandler(proxies) if proxies is not None \ else request.BaseHandler() opener = request.build_opener(handler) request.install_opener(opener) built_path = os.path.join(here, 'static/version.built') is_built = visdom.__version__ == 'no_version_file' if os.path.exists(built_path): with open(built_path, 'r') as build_file: build_version = build_file.read().strip() if build_version == visdom.__version__: is_built = True else: os.remove(built_path) # download files one-by-one: for (key, val) in ext_files.items(): # set subdirectory: sub_dir = 'fonts' if '.js' in key: sub_dir = 'js' if '.css' in key: sub_dir = 'css' # download file: filename = '%s/static/%s/%s' % (install_dir, sub_dir, val) if not os.path.exists(filename) or not is_built: req = request.Request(key, headers={'User-Agent': 'Chrome/30.0.0.0'}) try: data = opener.open(req).read() with open(filename, 'wb') as fwrite: fwrite.write(data) except HTTPError as exc: logging.error('Error {} while downloading {}'.format( exc.code, key)) except URLError as exc: logging.error('Error {} while downloading {}'.format( exc.reason, key)) if not is_built: with open(built_path, 'w+') as build_file: build_file.write(visdom.__version__)
def open(self, url, data=None, method=None, login=True, async_get=True, async_wait=True, retjson=True): url = self.base_url + url if data: try: if method != 'PUT': data = parse.urlencode(data).encode(encoding='UTF8') else: data = data.encode('utf-8') except TypeError as ex: print(data) logger.debug(ex) pass req = request.Request(url, data) else: req = request.Request(url) if login: if not self.login_cookie: self.login() req.add_header("Cookie", self.login_cookie) for (k, v) in self.headers: req.add_header(k, v) if method: req.get_method = lambda: method try: resp = request.urlopen(req) self.resp = resp rdata = resp.read() except HTTPError as e: rdata = e.fp.read() except URLError as e: self.status_code = 500 self.status = "ERROR" self.output = str(e) return except HTTPException as e: if six.PY3: rdata = e.args[0] else: raise e if not retjson: logger.debug(dict(resp)) self.data = rdata self.status_code = resp.code self.status = "OK" if resp.code == 200 else "ERROR" self.output = self.data return self.data #print(type(rdata)) if isinstance(rdata, bytes): rdata = rdata.decode('utf-8') rdata_reg = re.search('\{.+\}', rdata) rdata = rdata if rdata_reg is None else rdata_reg.group() try: rdata = json.loads(rdata) self.data = rdata self.status = self.data["status"] self.status_code = self.data["status_code"] self.output = self.data["output"] except ValueError as exc: print('ValueError:', exc, ' ; rdata:', rdata) self.data = rdata try: self.status_code = self.data["status_code"] self.status = self.data["status"] except Exception as exc: print(exc) logger.debug("exe : %s" % exc) logger.debug("rdata : %s" % rdata) try: self.status_code = rdata["status_code"] self.status = rdata["status"] except Exception as ex: self.status_code = 200 self.status = 200 self.status = "unknown" self.output = self.data if self.status_code == 200: self.status = "OK" if self.status_code == 201 and async_get: # It's a async task ! self.async_wait_time = async_wait_time time.sleep(async_first_wait_time) if DEBUG_ASYNC: logger.debug("jump to async") return self.open(async_url + '/' + self.output) if self.status_code == 100 and async_wait and self.async_wait_time > 0: # async task is running ! time.sleep(1) self.async_wait_time -= 1 if DEBUG_ASYNC: logger.debug(" async retry") return self.open(async_url + '/' + self.output) if self.async_wait_time <= 0: logger.error("Error : aysnc connection time out !!!") self.async_wait_time = async_wait_time logger.debug("return status %s in( %s ) %s secs" % (self.status_code, async_wait, self.async_wait_time)) return self.data
def _get_data(self, path): req = request.Request(path) response = self._get_response(req) return response.read()
def _http_get(url): req = request.Request(url) req.add_header('User-Agent', 'Mozilla/5.0') return request.urlopen(req, cafile=certifi.where()).read()
def post(self, path, postdata=None, files=None, ssl=False, compress=True, is_retry=False, use_ordered_dict=False, add_token=True, verify_cert=True, token=None, **kwargs): """ Returns result of an HTTP POST. Supports Multipart requests.""" #if path.find(" ") > -1: path = quote(path, ':/%') out_folder = kwargs.pop('out_folder', tempfile.gettempdir()) url = path if url.lower().find("https://") > -1 or\ url.lower().find("http://") > -1: url = path elif len(url) == 0: url = self.baseurl elif (len(url) > 0 and url[0] == '/' ) == False and \ self.baseurl.endswith('/') == False: url = "/{path}".format(path=url) if not url.startswith('http://') and \ not url.startswith('https://'): url = self.baseurl + url if ssl or self.all_ssl: url = url.replace('http://', 'https://') if verify_cert == False: import ssl ssl._create_default_https_context = ssl._create_unverified_context # Add the token if logged in if add_token: if self.is_logged_in: postdata['token'] = self.token if token: postdata['token'] = token if _log.isEnabledFor(logging.DEBUG): msg = 'REQUEST: ' + url + ', ' + str(postdata) if files: msg += ', files=' + str(files) _log.debug(msg) # If there are files present, send a multipart request if files: #parsed_url = urlparse(url) mpf = MultiPartForm(param_dict=postdata, files=files) req = request.Request(url) body = mpf.make_result req.add_header('User-agent', self._useragent) req.add_header('Content-type', mpf.get_content_type()) req.add_header('Content-length', len(body)) req.data = body headers = [('User-Agent', self._useragent), ('Content-type', mpf.get_content_type()), ('Content-length', len(body))] if self._referer and \ self._auth.lower() != 'pki': headers.append(('Referer', self._referer)) if compress: headers.append(('Accept-encoding', 'gzip')) if self._handlers is None: self._handlers = self.get_handlers() #handlers = self.get_handlers() opener = request.build_opener(*self._handlers) opener.addheaders = headers resp = opener.open(req) resp_data = self._process_response(resp) # Otherwise send a normal HTTP POST request else: encoded_postdata = None if postdata: postdata = {k: jsonize_dict(v) for k, v in postdata.items()} encoded_postdata = urlencode(postdata) headers = [('User-Agent', self._useragent)] if self._referer and \ self._auth.lower() != 'pki': headers.append(('Referer', self._referer)) if compress: headers.append(('Accept-encoding', 'gzip')) if self._handlers is None: self._handlers = self.get_handlers() opener = request.build_opener(*self._handlers) opener.addheaders = headers #request.install_opener(opener) req = request.Request(url, data=encoded_postdata.encode('utf-8'), headers={i[0]: i[1] for i in headers}) resp = opener.open(req) #request.urlopen(req) resp_data = self._process_response(resp, out_folder=out_folder, file_name=None) # Parse the response into JSON if _log.isEnabledFor(logging.DEBUG): _log.debug('RESPONSE: ' + url + ', ' + resp_data) #print(resp_data); if use_ordered_dict: resp_json = json.loads(resp_data, object_pairs_hook=OrderedDict) else: resp_json = json.loads(resp_data) # Check for errors, and handle the case where the token timed out # during use (and simply needs to be re-generated) try: if 'error' in resp_json or \ ('status' in resp_json and \ resp_json.get('status', None) != "success"): errorcode = resp_json['code'] if 'code' in resp_json else 0 if errorcode == 498 and not is_retry: _log.info( 'Token expired during post request, fetching a new ' + 'token and retrying') self.logout() newtoken = self.relogin() postdata['token'] = newtoken return self.post(path, postdata, files, ssl, compress, is_retry=True) elif errorcode == 498: raise RuntimeError('Invalid token') elif errorcode == 403: message = resp_json['error'][ 'message'] if 'message' in resp_json['error'] else '' if message == "SSL Required": return self.post(path, postdata, files, ssl=True, compress=compress, token=token, verify_cert=verify_cert, is_retry=True) if 'status' in resp_json: self._handle_json_error(resp_json, errorcode) else: self._handle_json_error(resp_json['error'], errorcode) return None except AttributeError: # Top-level JSON object isnt a dict, so can't have an error pass return resp_json
def open_url(url, data=None, headers=None, method=None, use_proxy=True, force=False, last_mod_time=None, timeout=10, validate_certs=True, url_username=None, url_password=None, http_agent=None, force_basic_auth=False, follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None): ''' Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3) Does not require the module environment ''' handlers = [] ssl_handler = maybe_add_ssl_handler(url, validate_certs) if ssl_handler: handlers.append(ssl_handler) parsed = generic_urlparse(urlparse(url)) if parsed.scheme != 'ftp': username = url_username if headers is None: headers = {} if username: password = url_password netloc = parsed.netloc elif '@' in parsed.netloc: credentials, netloc = parsed.netloc.split('@', 1) if ':' in credentials: username, password = credentials.split(':', 1) else: username = credentials password = '' parsed_list = parsed.as_list() parsed_list[1] = netloc # reconstruct url without credentials url = urlunparse(parsed_list) if username and not force_basic_auth: passman = urllib_request.HTTPPasswordMgrWithDefaultRealm() # this creates a password manager passman.add_password(None, netloc, username, password) # because we have put None at the start it will always # use this username/password combination for urls # for which `theurl` is a super-url authhandler = urllib_request.HTTPBasicAuthHandler(passman) digest_authhandler = urllib_request.HTTPDigestAuthHandler(passman) # create the AuthHandler handlers.append(authhandler) handlers.append(digest_authhandler) elif username and force_basic_auth: headers["Authorization"] = basic_auth_header(username, password) else: try: rc = netrc.netrc(os.environ.get('NETRC')) login = rc.authenticators(parsed.hostname) except IOError: login = None if login: username, _, password = login if username and password: headers["Authorization"] = basic_auth_header( username, password) if not use_proxy: proxyhandler = urllib_request.ProxyHandler({}) handlers.append(proxyhandler) if HAS_SSLCONTEXT and not validate_certs: # In 2.7.9, the default context validates certificates context = SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 context.options |= ssl.OP_NO_SSLv3 context.verify_mode = ssl.CERT_NONE context.check_hostname = False handlers.append( HTTPSClientAuthHandler(client_cert=client_cert, client_key=client_key, context=context)) elif client_cert: handlers.append( HTTPSClientAuthHandler(client_cert=client_cert, client_key=client_key)) # pre-2.6 versions of python cannot use the custom https # handler, since the socket class is lacking create_connection. # Some python builds lack HTTPS support. if hasattr(socket, 'create_connection') and CustomHTTPSHandler: handlers.append(CustomHTTPSHandler) handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs)) # add some nicer cookie handling if cookies is not None: handlers.append(urllib_request.HTTPCookieProcessor(cookies)) opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener) data = to_bytes(data, nonstring='passthru') if method: if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'PATCH'): raise ConnectionError('invalid HTTP request method; %s' % method.upper()) request = RequestWithMethod(url, method.upper(), data) else: request = urllib_request.Request(url, data) # add the custom agent header, to help prevent issues # with sites that block the default urllib agent string if http_agent: request.add_header('User-agent', http_agent) # Cache control # Either we directly force a cache refresh if force: request.add_header('cache-control', 'no-cache') # or we do it if the original is more recent than our copy elif last_mod_time: tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000') request.add_header('If-Modified-Since', tstamp) # user defined headers now, which may override things we've set above if headers: if not isinstance(headers, dict): raise ValueError("headers provided to fetch_url() must be a dict") for header in headers: request.add_header(header, headers[header]) urlopen_args = [request, None] if sys.version_info >= (2, 6, 0): # urlopen in python prior to 2.6.0 did not # have a timeout parameter urlopen_args.append(timeout) r = urllib_request.urlopen(*urlopen_args) return r
def request(self, path, request, body=None, **kwargs): """Make an HTTP request and return the results. :param path: Path used with the initialized URL to make a request. :param request: HTTP request type (GET, POST, PUT, DELETE). :param body: HTTP body of request. :key accept: Set HTTP 'Accept' header with this value. :key base_path: Override the base_path for this request. :key content: Set HTTP 'Content-Type' header with this value. """ out_hdrs = dict.copy(self.headers) if kwargs.get("accept"): out_hdrs['accept'] = kwargs.get("accept") if body: if isinstance(body, dict): body = six.text_type(jsonutils.dumps(body)) if body and len(body): out_hdrs['content-length'] = len(body) zfssaurl = self._path(path, kwargs.get("base_path")) req = urlrequest.Request(zfssaurl, body, out_hdrs) req.get_method = lambda: request maxreqretries = kwargs.get("maxreqretries", 10) retry = 0 response = None log_debug_msg(self, 'Request: %s %s' % (request, zfssaurl)) log_debug_msg(self, 'Out headers: %s' % out_hdrs) if body and body != '': log_debug_msg(self, 'Body: %s' % body) while retry < maxreqretries: try: response = urlrequest.urlopen(req, timeout=self.timeout) except urlerror.HTTPError as err: if err.code == http_client.NOT_FOUND: log_debug_msg(self, 'REST Not Found: %s' % err.code) else: log_debug_msg(self, ('REST Not Available: %s') % err.code) if (err.code == http_client.SERVICE_UNAVAILABLE and retry < maxreqretries): retry += 1 time.sleep(1) log_debug_msg(self, ('Server Busy retry request: %s') % retry) continue if ((err.code == http_client.UNAUTHORIZED or err.code == http_client.INTERNAL_SERVER_ERROR) and '/access/v1' not in zfssaurl): try: log_debug_msg(self, ('Authorizing request: ' '%(zfssaurl)s' 'retry: %(retry)d .') % { 'zfssaurl': zfssaurl, 'retry': retry }) self._authorize() req.add_header('x-auth-session', self.headers['x-auth-session']) except RestClientError: log_debug_msg(self, ('Cannot authorize.')) retry += 1 time.sleep(1) continue return RestResult(self.log_function, err=err) except urlerror.URLError as err: log_debug_msg(self, ('URLError: %s') % err.reason) raise RestClientError(-1, name="ERR_URLError", message=err.reason) break if ((response and response.getcode() == http_client.SERVICE_UNAVAILABLE) and retry >= maxreqretries): raise RestClientError(response.getcode(), name="ERR_HTTPError", message="REST Not Available: Disabled") return RestResult(self.log_function, response=response)
def destral(modules, tests, all_tests=None, enable_coverage=None, report_coverage=None, report_junitxml=None, dropdb=None, requirements=None, **kwargs): enable_lint = kwargs.pop('enable_lint') sys.argv = sys.argv[:1] service = OpenERPService() if report_junitxml: os.environ['DESTRAL_JUNITXML'] = report_junitxml else: report_junitxml = os.environ.get('DESTRAL_JUNITXML', False) if report_junitxml: junitxml_directory = os.path.abspath(report_junitxml) if not os.path.isdir(junitxml_directory): os.makedirs(junitxml_directory) if not modules: ci_pull_request = os.environ.get('CI_PULL_REQUEST') token = os.environ.get('GITHUB_TOKEN') repository = os.environ.get('CI_REPO') if ci_pull_request and token and repository: try: int(ci_pull_request) except: # If CI_PULL_REQUEST contains URL instead of PR number, get it ci_pull_request = ci_pull_request.split('/')[-1] url = 'https://api.github.com/repos/{repo}/pulls/{pr_number}'.format( repo=repository, pr_number=ci_pull_request) req = urllib2.Request(url, headers={ 'Authorization': 'token {0}'.format(token), 'Accept': 'application/vnd.github.patch' }) f = urllib2.urlopen(req) paths = find_files(f.read()) logger.info('Files from Pull Request: {0}: {1}'.format( ci_pull_request, ', '.join(paths))) else: paths = subprocess.check_output( ["git", "diff", "--name-only", "HEAD~1..HEAD"]) paths = [x for x in paths.split('\n') if x] logger.info('Files from last commit: {}'.format(', '.join(paths))) modules_to_test = [] for path in paths: module = detect_module(path) if module and module not in modules_to_test: modules_to_test.append(module) else: modules_to_test = modules[:] results = [] addons_path = service.config['addons_path'] root_path = service.config['root_path'] if not modules_to_test: coverage_config = {'source': [root_path], 'omit': ['*/addons/*/*']} else: coverage_config = { 'source': coverage_modules_path(modules_to_test, addons_path), 'omit': ['*/__terp__.py'] } coverage = OOCoverage(**coverage_config) coverage.enabled = (enable_coverage or report_coverage) junitxml_suites = [] coverage.start() server_spec_suite = get_spec_suite(root_path) if server_spec_suite: logging.info('Spec testing for server') report = run_spec_suite(server_spec_suite) results.append(not len(report.failed_examples) > 0) if report_junitxml: junitxml_suites += report.create_report_suites() coverage.stop() logger.info('Modules to test: {}'.format(','.join(modules_to_test))) for module in modules_to_test: with RestorePatchedRegisterAll(): if requirements: install_requirements(module, addons_path) spec_suite = get_spec_suite(os.path.join(addons_path, module)) if spec_suite: logger.info('Spec testing module %s', module) coverage.start() report = run_spec_suite(spec_suite) coverage.stop() results.append(not len(report.failed_examples) > 0) if report_junitxml: junitxml_suites += report.create_report_suites() logger.info('Unit testing module %s', module) os.environ['DESTRAL_MODULE'] = module coverage.start() suite = get_unittest_suite(module, tests) suite.drop_database = dropdb suite.config['all_tests'] = all_tests if all_tests: for m in get_dependencies(module, addons_path): for test in get_unittest_suite(m): if test not in suite: suite.addTest(test) result = run_unittest_suite(suite) coverage.stop() results.append(result.wasSuccessful()) if report_junitxml: junitxml_suites.append(result.get_test_suite(module)) if report_junitxml: from junit_xml import TestSuite for suite in junitxml_suites: with open(os.path.join(report_junitxml, suite.name + '.xml'), 'w') as report_file: report_file.write(TestSuite.to_xml_string([suite])) logger.info('Saved report XML on {}/'.format(report_junitxml)) if report_coverage: coverage.report() if enable_coverage: coverage.save() if enable_lint: modules_path = [ '{}/{}'.format(addons_path, m) for m in modules_to_test ] if modules_path: run_linter(modules_path) if not all(results): sys.exit(1)
def get(self, endpoint=None, cookie=None): req = request.Request(self.url + endpoint) if cookie: req.add_header('cookie', cookie) return self.opener.open(req)
def request(url, endpoint=IDP_ENDPOINTS['LIGO.ORG'], use_kerberos=None, debug=False): """Request the given URL using ECP shibboleth authentication This requires an active Kerberos ticket for the user, to get one: >>> from ligo.org import kinit >>> kinit('albert.einstein') Then request as follows >>> from ligo.org import request >>> response = request(myurl) >>> print(response.read()) Adapted from https://wiki.shibboleth.net/confluence/download/attachments/4358416/ecp.py Parameters ---------- url : `str` URL path for request endpoint : `str` ECP endpoint URL for request use_kerberos : `bool`, optional use existing kerberos credential for login, default is to try, but fall back to username/password prompt debug : `bool`, optional, default: `False` query in verbose debugging mode Returns ------- response : `str` the raw (decoded) response from the URL, probably XML/HTML or JSON """ login_host = urlparse(endpoint).netloc # create a cookie jar and cookie handler (and read existing cookies) cookie_jar = ECPCookieJar() if os.path.exists(COOKIE_JAR): try: cookie_jar.load(COOKIE_JAR, ignore_discard=True) except http_cookiejar.LoadError as e: warnings.warn('Caught error loading ECP cookie: %s' % str(e)) cookie_handler = urllib_request.HTTPCookieProcessor(cookie_jar) # need an instance of HTTPS handler to do HTTPS httpsHandler = urllib_request.HTTPSHandler(debuglevel=0) if debug: httpsHandler.set_http_debuglevel(1) # create the base opener object opener = urllib_request.build_opener(cookie_handler, httpsHandler) # get kerberos credentials if available if use_kerberos is None: try: creds = klist() except KerberosError: use_kerberos = False else: if creds: use_kerberos = True else: use_kerberos = False if use_kerberos: opener.add_handler( HTTPNegotiateAuthHandler(service_principal='HTTP@%s' % login_host)) # -- intiate ECP request -------------------- # headers needed to indicate to the SP an ECP request headers = { 'Accept': 'text/html; application/vnd.paos+xml', 'PAOS': 'ver="urn:liberty:paos:2003-08";' '"urn:oasis:names:tc:SAML:2.0:profiles:SSO:ecp"', } # request target from SP request = urllib_request.Request(url=url, headers=headers) response = opener.open(request) # convert the SP resonse from string to etree Element object sp_response = etree.XML(response.read()) # pick out the relay state element from the SP so that it can # be included later in the response to the SP namespaces = { 'ecp': 'urn:oasis:names:tc:SAML:2.0:profiles:SSO:ecp', 'S': 'http://schemas.xmlsoap.org/soap/envelope/', 'paos': 'urn:liberty:paos:2003-08' } relay_state = sp_response.xpath("//ecp:RelayState", namespaces=namespaces)[0] # make a deep copy of the SP response and then remove the header # in order to create the package for the IdP idp_request = deepcopy(sp_response) header = idp_request[0] idp_request.remove(header) # -- authenticate with endpoint ------------- request = urllib_request.Request(endpoint, data=etree.tostring(idp_request)) request.get_method = lambda: 'POST' request.add_header('Content-Type', 'test/xml; charset=utf-8') # get credentials for non-kerberos request if not use_kerberos: # prompt the user for a password login = input("Enter username for %s: " % login_host) password = getpass.getpass("Enter password for login '%s': " % login) # combine the login and password, base64 encode, and send # using the Authorization header base64string = base64.encodestring( ('%s:%s' % (login, password)).encode()).decode().replace('\n', '') request.add_header('Authorization', 'Basic %s' % base64string) response = opener.open(request) idp_response = etree.XML(response.read()) assertion_consumer_service = idp_response.xpath( "/S:Envelope/S:Header/ecp:Response/@AssertionConsumerServiceURL", namespaces=namespaces)[0] # make a deep copy of the IdP response and replace its # header contents with the relay state initially sent by # the SP sp_package = deepcopy(idp_response) sp_package[0][0] = relay_state headers = {'Content-Type': 'application/vnd.paos+xml'} # POST the package to the SP request = urllib_request.Request(url=assertion_consumer_service, data=etree.tostring(sp_package), headers=headers) request.get_method = lambda: 'POST' response = opener.open(request) # -- cache cookies -------------------------- cookie_jar.save(COOKIE_JAR, ignore_discard=True) # -- actually send GET ---------------------- request = urllib_request.Request(url=url) response = opener.open(request) return response.read()
def get(self, endpoint): req = request.Request(self.url + endpoint) return self._open(req)
def _request_compute_metadata(path): req = request.Request('%s/computeMetadata/v1/%s' % (_gce_metadata_endpoint(), path), headers={'Metadata-Flavor': 'Google'}) resp = request.urlopen(req) return _as_text(resp.read())
def urlopen(url, *args, **kwargs): """Wrapper around urlopen. Same function as 'urlopen', but with the ability to define headers. """ request = urlrequest.Request(url, headers=HEADERS) return urlrequest.urlopen(request, *args, **kwargs)