def get_canonical_and_alternates_urls(url, drop_ln=True, washed_argd=None, quote_path=False): """ Given an Invenio URL returns a tuple with two elements. The first is the canonical URL, that is the original URL with CFG_SITE_URL prefix, and where the ln= argument stripped. The second element element is mapping, language code -> alternate URL @param quote_path: if True, the path section of the given C{url} is quoted according to RFC 2396 """ dummy_scheme, dummy_netloc, path, dummy_params, query, fragment = urlparse(url) canonical_scheme, canonical_netloc = urlparse(cfg.get("CFG_SITE_URL"))[0:2] parsed_query = washed_argd or parse_qsl(query) no_ln_parsed_query = [(key, value) for (key, value) in parsed_query if key != "ln"] if drop_ln: canonical_parsed_query = no_ln_parsed_query else: canonical_parsed_query = parsed_query if quote_path: path = urllib.quote(path) canonical_query = urlencode(canonical_parsed_query) canonical_url = urlunparse((canonical_scheme, canonical_netloc, path, dummy_params, canonical_query, fragment)) alternate_urls = {} for ln in cfg.get("CFG_SITE_LANGS"): alternate_query = urlencode(no_ln_parsed_query + [("ln", ln)]) alternate_url = urlunparse((canonical_scheme, canonical_netloc, path, dummy_params, alternate_query, fragment)) alternate_urls[ln] = alternate_url return canonical_url, alternate_urls
def _secure_request(self, url, method, data=None, files=None, headers=None, raw=False, send_as_json=True, content_type=None, **request_kwargs): full_url = self.build_url(url) # Add token (if it's not already there) if self._token: parsed = list(urlparse(full_url)) if not parsed[4]: # query parsed[4] = 'token=%s' % self._token full_url = urlunparse(parsed) elif 'token' not in parse_qs(parsed[4]): parsed[4] += '&token=%s' % self._token full_url = urlunparse(parsed) headers = headers or {} # If files are being sent, we cannot encode data as JSON if send_as_json and not files: headers['content-type'] = 'application/json' data = json.dumps(data or {}) else: if content_type: headers['content-type'] = content_type data = data or '' method = getattr(requests, method, None) response = method(full_url, data=data, files=files, headers=headers, **request_kwargs) self.check_for_errors(response) # Raise exception if something failed if raw or not response.content: return response.content return json.loads(response.text)
def get_commit_url(commit, pkg): try: upstream_url = parse.urlsplit(pkg["upstream"]) if upstream_url.netloc == "git.openstack.org": commit_url = ("http", upstream_url.netloc, "/cgit%s/commit/?id=" % upstream_url.path, "", "", "") commit_url = parse.urlunparse(commit_url) elif upstream_url.netloc == "github.com": commit_url = ("https", upstream_url.netloc, "%s/commit/" % upstream_url.path, "", "", "") commit_url = parse.urlunparse(commit_url) else: # Fallback when no cgit URL can be defined commit_url = pkg["upstream"] except KeyError: # This should not happen, but pkg['upstream'] may not be present # after some error in the gitrepo driver commit_url = '' return commit_url
def base_url(self, filters, auth_data=None): """Base URL from catalog Filters can be: - service: compute, image, etc - region: the service region - endpoint_type: adminURL, publicURL, internalURL - api_version: replace catalog version with this - skip_path: take just the base URL """ if auth_data is None: auth_data = self.auth_data token, _auth_data = auth_data service = filters.get('service') region = filters.get('region') endpoint_type = filters.get('endpoint_type', 'publicURL') if service is None: raise exceptions.EndpointNotFound("No service provided") _base_url = None for ep in _auth_data['serviceCatalog']: if ep["type"] == service: for _ep in ep['endpoints']: if region is not None and _ep['region'] == region: _base_url = _ep.get(endpoint_type) if not _base_url: # No region matching, use the first _base_url = ep['endpoints'][0].get(endpoint_type) break if _base_url is None: raise exceptions.EndpointNotFound( "service: %s, region: %s, endpoint_type: %s" % (service, region, endpoint_type)) parts = urlparse.urlparse(_base_url) if filters.get('api_version', None) is not None: version_path = '/%s' % filters['api_version'] path = re.sub(r'(^|/)+v\d+(?:\.\d+)?', version_path, parts.path, count=1) _base_url = urlparse.urlunparse((parts.scheme, parts.netloc, path or version_path, parts.params, parts.query, parts.fragment)) if filters.get('skip_path', None) is not None and parts.path != '': _base_url = urlparse.urlunparse((parts.scheme, parts.netloc, '/', parts.params, parts.query, parts.fragment)) return _base_url
def output(self, key, obj): try: data = to_marshallable_type(obj) endpoint = self.endpoint if self.endpoint is not None else request.endpoint o = urlparse(url_for(endpoint, _external=self.absolute, **data)) if self.absolute: scheme = self.scheme if self.scheme is not None else o.scheme return urlunparse((scheme, o.netloc, o.path, "", "", "")) return urlunparse(("", "", o.path, "", "", "")) except TypeError as te: raise MarshallingError(te)
def decorate(self, *args, **kwargs): try: return func(self, *args, **kwargs) except exc.OAuthException as o_exc: # Extract the parameters error = o_exc.error error_description = o_exc.msg or _("No details available.") # If we have a redirect URL, build the error redirect. if o_exc.redirect_uri: # Split the redirect_url apart parts = urlparse(o_exc.redirect_uri) # Add the error and error_description if parts.query: params = urlparse.parse_qsl(parts.query) else: params = [] params.append(('error', error)) params.append(('error_description', error_description)) # Overwrite the old query params and reconstruct the URL parts_list = list(parts) parts_list[4] = urlencode(params) location = urlunparse(parts_list) redirect(location) else: error_body = { 'error': error, 'error_description': error_description } response.json = error_body abort(o_exc.code, error_description, json_body=error_body)
def POST(self, endpoint, data=None): if data is None: data = dict() response = requests.post(urlunparse((self.scheme, self.host, endpoint, '', '', '')), headers={'Content-Type': 'application/json'}, json=data, auth=self.auth) response.raise_for_status() return makeresponse(response)
def url_add_parameters(url, params): """Adds parameters to URL, parameter will be repeated if already present""" if params: fragments = list(urlparse(url)) fragments[4] = urlencode(parse_qsl(fragments[4]) + params.items()) url = urlunparse(fragments) return url
def _parse_contents(self, response): # Wix pages aren't really parseable, so anytime we see them, # let's re-run it (depth-1) with an escaped-fragment to get the real html source if 'https://static.wixstatic.com/' in response.body and '_escaped_fragment_' not in response.url: parsed_url = urlparse(response.url) qs = parse_qs(parsed_url.query) qs['_escaped_fragment_'] = '' wix_scrapeable_url = urlunparse( (parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, urlencode(qs), parsed_url.fragment) ) response.meta['depth'] -= 1 return [scrapy.Request(wix_scrapeable_url, self.parse)] return if not hasattr(response, 'selector'): logging.info('Skipping unknown file from: %s', response.url) return # Get all text contents of tags (unless they are script or style tags) text_contents = ' '.join(response.selector.xpath('//*[not(self::script|self::style)]/text()').extract()).lower() processed_text = grammar_matcher.StringProcessor(text_contents, regex_keywords.WORD_BOUNDARIES) wrong = processed_text.get_tokens(all_styles.DANCE_WRONG_STYLE) good = processed_text.get_tokens(rules.STREET_STYLE) if (wrong or good): #print response.url, set(wrong), set(good) pass
def _build_url(url, _params): """Build the actual URL to use.""" # Support for unicode domain names and paths. scheme, netloc, path, params, query, fragment = urlparse(url) netloc = netloc.encode('idna').decode('utf-8') if not path: path = '/' if six.PY2: if isinstance(scheme, six.text_type): scheme = scheme.encode('utf-8') if isinstance(netloc, six.text_type): netloc = netloc.encode('utf-8') if isinstance(path, six.text_type): path = path.encode('utf-8') if isinstance(params, six.text_type): params = params.encode('utf-8') if isinstance(query, six.text_type): query = query.encode('utf-8') if isinstance(fragment, six.text_type): fragment = fragment.encode('utf-8') enc_params = _encode_params(_params) if enc_params: if query: query = '%s&%s' % (query, enc_params) else: query = enc_params url = (urlunparse([scheme, netloc, path, params, query, fragment])) return url
def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, encoding=None): """Canonicalize the given url by applying the following procedures: - sort query arguments, first by key, then by value - percent encode paths and query arguments. non-ASCII characters are percent-encoded using UTF-8 (RFC-3986) - normalize all spaces (in query arguments) '+' (plus symbol) - normalize percent encodings case (%2f -> %2F) - remove query arguments with blank values (unless keep_blank_values is True) - remove fragments (unless keep_fragments is True) The url passed can be a str or unicode, while the url returned is always a str. For examples see the tests in tests/test_utils_url.py """ scheme, netloc, path, params, query, fragment = parse_url(url) keyvals = parse_qsl(query, keep_blank_values) keyvals.sort() query = urlencode(keyvals) # XXX: copied from w3lib.url.safe_url_string to add encoding argument # path = to_native_str(path, encoding) # path = moves.urllib.parse.quote(path, _safe_chars, encoding='latin1') or '/' path = safe_url_string(_unquotepath(path)) or '/' fragment = '' if not keep_fragments else fragment return urlunparse((scheme, netloc.lower(), path, params, query, fragment))
def to_url(self): """Serialize as a URL for a GET request.""" base_url = urlparse.urlparse(self.url) try: query = base_url.query except AttributeError: # must be python <2.5 query = base_url[4] query = parse_qs(query) for k, v in self.items(): if isinstance(v, six.text_type): v = v.encode("utf-8") query.setdefault(k, []).append(v) try: scheme = base_url.scheme netloc = base_url.netloc path = base_url.path params = base_url.params fragment = base_url.fragment except AttributeError: # must be python <2.5 scheme = base_url[0] netloc = base_url[1] path = base_url[2] params = base_url[3] fragment = base_url[5] url = (scheme, netloc, path, params, urllib.urlencode(query, True), fragment) return urlparse.urlunparse(url)
def get(self, request, *args, **kwargs): validate_session_for_mturk(request, self.session) mturk_settings = self.session.config['mturk_hit_settings'] initial = { 'title': mturk_settings['title'], 'description': mturk_settings['description'], 'keywords': ', '.join(mturk_settings['keywords']), 'money_reward': self.session.config['participation_fee'], 'in_sandbox': settings.DEBUG, 'minutes_allotted_per_assignment': ( mturk_settings['minutes_allotted_per_assignment'] ), 'expiration_hours': mturk_settings['expiration_hours'], 'assignments': self.session.mturk_num_participants, } form = self.get_form(initial=initial) context = self.get_context_data(form=form) context['mturk_enabled'] = ( bool(settings.AWS_ACCESS_KEY_ID) and bool(settings.AWS_SECRET_ACCESS_KEY) ) context['runserver'] = 'runserver' in sys.argv url = self.request.build_absolute_uri( reverse('MTurkCreateHIT', args=(self.session.code,)) ) secured_url = urlunparse(urlparse(url)._replace(scheme='https')) context['secured_url'] = secured_url return self.render_to_response(context)
def get_temp_url(self, container, object, timeout): """Returns the temp url for the given Swift object. :param container: The name of the container in which Swift object is placed. :param object: The name of the Swift object. :param timeout: The timeout in seconds after which the generated url should expire. :returns: The temp url for the object. :raises: SwiftOperationError, if any operation with Swift fails. """ try: account_info = self.connection.head_account() except swift_exceptions.ClientException as e: operation = _("head account") raise exception.SwiftOperationError(operation=operation, error=e) storage_url, token = self.connection.get_auth() parse_result = parse.urlparse(storage_url) swift_object_path = '/'.join((parse_result.path, container, object)) temp_url_key = account_info['x-account-meta-temp-url-key'] url_path = swift_utils.generate_temp_url(swift_object_path, timeout, temp_url_key, 'GET') return parse.urlunparse((parse_result.scheme, parse_result.netloc, url_path, None, None, None))
def update_content( article, content, siteurl ): self = article stripped_content = content.strip() if not stripped_content.startswith( PREFIX ): return content # remove {filename} url = stripped_content[ len( PREFIX ): ] parse_result = urlparse( url ) path = parse_result.path if path.startswith( '/' ): path = path[ 1: ] else: # relative link path = self.get_relative_source_path( os.path.join( self.relative_dir, path ) ) # unescape spaces if necessary if path not in self._context[ 'filenames' ]: path = path.replace( '%20', ' ' ) if path not in self._context[ 'filenames' ]: logger.warning( "Unable to find `%s`, skipping url replacement.", url, extra = { 'limit_msg': ( "Other resources were not found and their urls not replaced" ) } ) return content linked_content = self._context[ 'filenames' ][ path ] parts = list( parse_result ) parts[ 2 ] = '/'.join( [ siteurl, linked_content.url ] ) return urlunparse( parts )
def __init__(self, info_yaml, basedir=''): """Import the solution's info.yaml file.""" f, url_parts = self._open(info_yaml, basedir) solution_yaml = f.read().decode('utf-8') self.basedir = urlunparse((url_parts.scheme, url_parts.netloc, os.path.dirname(url_parts.path), None, None, None)) # create a markdown converter and modify it to rebase image links markdown = Markdown() markdown.inlinePatterns['image_link'] = _RebasedImageLinkPattern( self.basedir, IMAGE_LINK_RE, markdown) markdown.inlinePatterns['image_reference'] = _RebasedImageRefPattern( self.basedir, IMAGE_REFERENCE_RE, markdown) # import the solution's metadata info = yaml.load(solution_yaml) self.id = hashlib.md5(solution_yaml.encode('utf-8')).hexdigest() self.title = info['name'] self.release = str(info['release']) if 'logo' in info: self.logo = self._make_absolute_path(info.get('logo'), self.basedir)[0] # in all the following fields, newlines are suppressed because they # are not rendered properly in Javascript strings by Django self.short_description = \ markdown.convert(info['short_desc']).replace('\n', '') self.long_description = \ markdown.convert(info['long_desc']).replace('\n', '') self.architecture = \ markdown.convert(info['architecture']).replace('\n', '') self.design_specs = info.get('design_specs', []) self.heat_template = info['heat_template'] self.env_file = info.get('env_file') # environments are optional
def methodNext(self, previous_request, previous_response): """Retrieves the next page of results. Args: previous_request: The request for the previous page. (required) previous_response: The response from the request for the previous page. (required) Returns: A request object that you can call 'execute()' on to request the next page. Returns None if there are no more items in the collection. """ # Retrieve nextPageToken from previous_response # Use as pageToken in previous_request to create new request. if "nextPageToken" not in previous_response or not previous_response["nextPageToken"]: return None request = copy.copy(previous_request) pageToken = previous_response["nextPageToken"] parsed = list(urlparse(request.uri)) q = parse_qsl(parsed[4]) # Find and remove old 'pageToken' value from URI newq = [(key, value) for (key, value) in q if key != "pageToken"] newq.append(("pageToken", pageToken)) parsed[4] = urlencode(newq) uri = urlunparse(parsed) request.uri = uri logger.info("URL being requested: {0!s} {1!s}".format(methodName, uri)) return request
def configure_content(config): config.add_view(favicon_view, name='favicon.ico', http_cache=(86400, {'public': True})) config.registry.skip_logging.add('/favicon.ico') config.add_view(robotstxt_view, name='robots.txt', http_cache=(86400, {'public': True})) config.registry.skip_logging.add('/robots.txt') config.add_view(touchicon_view, name='apple-touch-icon-precomposed.png', http_cache=(86400, {'public': True})) config.registry.skip_logging.add('/apple-touch-icon-precomposed.png') config.add_static_view( name='static', path='ichnaea.content:static', cache_max_age=86400) # BBB: leaders/leaders_weekly redirect to new service config.add_route('leaders_weekly', '/leaders/weekly') config.add_route('leaders', '/leaders') config.add_route('stats_regions', '/stats/regions') config.add_route('stats', '/stats') config.scan('ichnaea.content.views') assets_url = config.registry.settings.get('assets', {}).get('url', None) config.registry.tiles_url = tiles_url = map_tiles_url(assets_url) result = urlparse.urlsplit(tiles_url) tiles = urlparse.urlunparse((result.scheme, result.netloc, '', '', '', '')) config.registry.csp = CSP_POLICY.format(base=CSP_BASE, tiles=tiles)
def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@" - ``strip_default_port`` removes ":80" (resp. ":443", ":21") from http:// (resp. https://, ftp://) URLs - ``origin_only`` replaces path component with "/", also dropping query and fragment components ; it also strips credentials - ``strip_fragment`` drops any #fragment component """ parsed_url = urlparse(url) netloc = parsed_url.netloc if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1] if strip_default_port and parsed_url.port: if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443), ('ftp', 21)): netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') return urlunparse(( parsed_url.scheme, netloc, '/' if origin_only else parsed_url.path, '' if origin_only else parsed_url.params, '' if origin_only else parsed_url.query, '' if strip_fragment else parsed_url.fragment ))
def __init__(self, requirement_string): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: raise InvalidRequirement( 'Parse error at "{0!r}": {1}'.format( requirement_string[e.loc : e.loc + 8], e.msg ) ) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) if parsed_url.scheme == "file": if urlparse.urlunparse(parsed_url) != req.url: raise InvalidRequirement("Invalid URL given") elif not (parsed_url.scheme and parsed_url.netloc) or ( not parsed_url.scheme and not parsed_url.netloc ): raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: self.url = None self.extras = set(req.extras.asList() if req.extras else []) self.specifier = SpecifierSet(req.specifier) self.marker = req.marker if req.marker else None
def relative_uri(source, target): """ Make a relative URI from source to target. """ su = urlparse.urlparse(source) tu = urlparse.urlparse(target) extra = list(tu[3:]) relative = None if tu[0] == '' and tu[1] == '': if tu[2] == su[2]: relative = '' elif not tu[2].startswith('/'): relative = tu[2] elif su[0:2] != tu[0:2]: return target if relative is None: if tu[2] == su[2]: relative = '' else: relative = os.path.relpath(tu[2], os.path.dirname(su[2])) if relative == '.': relative = '' relative = urlparse.urlunparse(["", "", relative] + extra) return relative
def create_plugin(self, session, version, url, raw_status=None): """Handle default Keystone endpoint configuration Build the actual API endpoint from the scheme, host and port of the original auth URL and the rest from the returned version URL. """ ver_u = urlparse.urlparse(url) # Only hack this if it is the default setting if ver_u.netloc.startswith('localhost'): auth_u = urlparse.urlparse(self.auth_url) # from original auth_url: scheme, netloc # from api_url: path, query (basically, the rest) url = urlparse.urlunparse(( auth_u.scheme, auth_u.netloc, ver_u.path, ver_u.params, ver_u.query, ver_u.fragment, )) LOG.debug('Version URL updated: %s' % url) return super(OSCGenericPassword, self).create_plugin( session=session, version=version, url=url, raw_status=raw_status, )
def merge(request, pageTree, removePanelLinks=False, removeLayoutLink=True): """Perform panel merging for the given page. Returns None if the page has no layout. """ # Find layout node layoutHref = utils.xpath1(utils.layoutXPath, pageTree) if layoutHref is None: return None # Resolve layout tree baseURL = request.getURL() if request.getVirtualRoot(): # plone.subrequest deals with VHM requests baseURL = '' layoutHref = parse.urljoin(baseURL, layoutHref) # noqa: turn the link absolute # Pass special ajax_load parameter forward to allow layout indirection # views to select, for example, default AJAX layout instead of full layout. if request.form.get('ajax_load'): parts = list(parse.urlparse(layoutHref)) query = parse.parse_qs(parts[4]) query['ajax_load'] = request.form.get('ajax_load') parts[4] = parse.urlencode(query) layoutHref = parse.urlunparse(parts) layoutTree = utils.resolve(layoutHref) if layoutTree is None: return None # Map page panels onto the layout pagePanels = dict( (node.attrib['data-panel'], node) for node in utils.panelXPath(pageTree) ) layoutPanels = dict( (node.attrib['data-panel'], node) for node in utils.panelXPath(layoutTree) ) # Site layout should always have element with data-panel="content" # Note: This could be more generic, but that would empower editors too much if 'content' in pagePanels and 'content' not in layoutPanels: for node in layoutTree.xpath('//*[@id="content"]'): node.attrib['data-panel'] = 'content' layoutPanels['content'] = node break for panelId, layoutPanelNode in layoutPanels.items(): pagePanelNode = pagePanels.get(panelId, None) if pagePanelNode is not None: utils.replace_content(layoutPanelNode, pagePanelNode) if removePanelLinks: del layoutPanelNode.attrib['data-panel'] if removeLayoutLink: del pageTree.getroot().attrib[utils.layoutAttrib] return layoutTree
def get_redirect_url(self, state="", redirect_uri=None, scope=None, action=None, email=None, client_id=None, code_challenge=None, code_challenge_method=None, access_type=None, keys_jwk=None): """Get the URL to redirect to to initiate the oauth flow.""" if client_id is None: client_id = self.client_id params = { "client_id": client_id, "state": state, } if redirect_uri is not None: params["redirect_uri"] = redirect_uri if scope is not None: params["scope"] = scope if action is not None: params["action"] = action if email is not None: params["email"] = email if code_challenge is not None: params["code_challenge"] = code_challenge if code_challenge_method is not None: params["code_challenge_method"] = code_challenge_method if keys_jwk is not None: params["keys_jwk"] = keys_jwk if access_type is not None: params["access_type"] = access_type query_str = urlencode(params) authorization_url = urlparse(self.server_url + "/authorization") return urlunparse(authorization_url._replace(query=query_str))
def build_url(base, additional_params=None): """Construct a URL based off of base containing all parameters in the query portion of base plus any additional parameters. :param base: Base URL :type base: str ::param additional_params: Additional query parameters to include. :type additional_params: dict :rtype: str """ url = urlparse(base) query_params = {} query_params.update(parse_qsl(url.query, True)) if additional_params is not None: query_params.update(additional_params) for k, v in six.iteritems(additional_params): if v is None: query_params.pop(k) return urlunparse((url.scheme, url.netloc, url.path, url.params, urlencode(query_params), url.fragment))
def check(self, instance): if not self.cadvisor_url: cadvisor_url = instance.get("cadvisor_url", None) detect_cadvisor_url = instance.get("kubernetes_detect_cadvisor", False) if not cadvisor_url: if detect_cadvisor_url: kubernetes_connector = utils.KubernetesConnector(self.connection_timeout) host = kubernetes_connector.get_agent_pod_host() cadvisor_url = "http://{}:4194".format(host) else: exception_message = "Either cAdvisor url or kubernetes " \ "detect cAdvisor must be set when " \ "monitoring a Kubernetes Node." self.log.error(exception_message) raise Exception(exception_message) self.cadvisor_url = "{}/{}".format(cadvisor_url, "api/v2.0/stats?count=1") dimensions = self._set_dimensions(None, instance) try: host_metrics = requests.get(self.cadvisor_url, self.connection_timeout).json() except Exception as e: self.log.error("Error communicating with cAdvisor to collect data - {}".format(e)) else: # Retrieve machine info only once if not self.cadvisor_machine_url: # Replace path in current cadvisor_url result = urlparse(self.cadvisor_url) self.cadvisor_machine_url = urlunparse(result._replace(path="api/v2.0/machine")) try: machine_info = requests.get(self.cadvisor_machine_url).json() except Exception as ex: self.log.error( "Error communicating with cAdvisor to collect machine data - {}".format(ex)) else: self._parse_machine_info(machine_info) self._parse_send_metrics(host_metrics, dimensions)
def replace_query_parameters(request, replacements): """ Replace query parameters in request according to replacements. The replacements should be a list of (key, value) pairs where the value can be any of: 1. A simple replacement string value. 2. None to remove the given header. 3. A callable which accepts (key, value, request) and returns a string value or None. """ query = request.query new_query = [] replacements = dict(replacements) for k, ov in query: if k not in replacements: new_query.append((k, ov)) else: rv = replacements[k] if callable(rv): rv = rv(key=k, value=ov, request=request) if rv is not None: new_query.append((k, rv)) uri_parts = list(urlparse(request.uri)) uri_parts[4] = urlencode(new_query) request.uri = urlunparse(uri_parts) return request
def _parseURL(url): try: url = urinorm.urinorm(url) except ValueError: return None proto, netloc, path, params, query, frag = urlparse(url) if not path: # Python <2.4 does not parse URLs with no path properly if not query and '?' in netloc: netloc, query = netloc.split('?', 1) path = '/' path = urlunparse(('', '', path, params, query, frag)) if ':' in netloc: try: host, port = netloc.split(':') except ValueError: return None if not re.match(r'\d+$', port): return None else: host = netloc port = '' host = host.lower() if not host_segment_re.match(host): return None return proto, host, port, path
def clean_ows_url(url): """ clean an OWS URL of basic service elements source: https://stackoverflow.com/a/11640565 """ if url is None or not url.startswith('http'): return url filtered_kvp = {} basic_service_elements = ('service', 'version', 'request') parsed = urlparse(url) qd = parse_qs(parsed.query, keep_blank_values=True) for key, value in qd.items(): if key.lower() not in basic_service_elements: filtered_kvp[key] = value newurl = urlunparse([ parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlencode(filtered_kvp, doseq=True), parsed.fragment ]) return newurl
def get_sample_data(self, meter_name, parse_url, params, cache): extractor = self._get_extractor(meter_name) if extractor is None: # The way to getting meter is not implemented in this driver or # OpenDaylight REST API has not api to getting meter. return None iter = self._get_iter(meter_name) if iter is None: # The way to getting meter is not implemented in this driver or # OpenDaylight REST API has not api to getting meter. return None parts = urlparse.ParseResult( params.get("scheme", ["http"])[0], parse_url.netloc, parse_url.path, None, None, None ) endpoint = urlparse.urlunparse(parts) data = self._prepare_cache(endpoint, params, cache) samples = [] for name, value in six.iteritems(data): timestamp = value["timestamp"] for sample in iter(extractor, value): if sample is not None: # set controller name and container name # to resource_metadata sample[2]["controller"] = "OpenDaylight" sample[2]["container"] = name samples.append(sample + (timestamp,)) return samples
def urlunparse(self, url_tuple): """Puts a url back together again, in the manner that urlparse breaks it. """ return parse.urlunparse(url_tuple)
def spec_url(spec_file, schema='file'): directory = path.abspath(path.dirname(__file__)) full_path = path.join(directory, spec_file) return urlunparse((schema, None, full_path, None, None, None))
def post(self, request, *args, **kwargs): form = self.get_form(data=request.POST, files=request.FILES) if not form.is_valid(): return self.form_invalid(form) session = self.session use_sandbox = 'use_sandbox' in form.data # session can't be created if (not self.in_public_domain(request, *args, **kwargs) and not use_sandbox): msg = ('<h1>Error: ' 'oTree must run on a public domain for Mechanical Turk' '</h1>') return HttpResponseServerError(msg) mturk_settings = session.config['mturk_hit_settings'] qualification_id = mturk_settings.get('grant_qualification_id', None) # verify that specified qualification type # for preventing retakes exists on mturk server url_landing_page = self.request.build_absolute_uri( reverse('MTurkLandingPage', args=(session.code, ))) # updating schema from http to https # this is compulsory for MTurk exteranlQuestion # TODO: validate, that the server support https # (heroku does support by default) secured_url_landing_page = urlunparse( urlparse(url_landing_page)._replace(scheme='https')) # TODO: validate that there is enough money for the hit money_reward = form.cleaned_data['money_reward'] # assign back to participation_fee, in case it was changed # in the form # need to convert back to RealWorldCurrency, because easymoney # MoneyFormField returns a decimal, not Money (not sure why) # see views.admin.EditSessionProperties session.config['participation_fee'] = RealWorldCurrency(money_reward) external_question = ''' <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"> <ExternalURL>{}</ExternalURL> <FrameHeight>{}</FrameHeight> </ExternalQuestion> '''.format(secured_url_landing_page, mturk_settings['frame_height']) qualifications = mturk_settings.get('qualification_requirements') if qualifications and not isinstance(qualifications[0], dict): raise ValueError( 'settings.py: You need to upgrade your MTurk qualification_requirements ' 'to the boto3 format. See the documentation.') mturk_hit_parameters = { 'Title': form.cleaned_data['title'], 'Description': form.cleaned_data['description'], 'Keywords': form.cleaned_data['keywords'], 'Question': external_question, 'MaxAssignments': form.cleaned_data['assignments'], 'Reward': str(float(money_reward)), 'QualificationRequirements': qualifications, 'AssignmentDurationInSeconds': 60 * form.cleaned_data['minutes_allotted_per_assignment'], 'LifetimeInSeconds': int(60 * 60 * form.cleaned_data['expiration_hours']), # prevent duplicate HITs 'UniqueRequestToken': 'otree_{}'.format(session.code), } with MTurkClient(use_sandbox=use_sandbox, request=request) as mturk_client: if qualification_id: try: mturk_client.get_qualification_type( QualificationTypeId=qualification_id) # it's RequestError, but except Exception as exc: if use_sandbox: sandbox_note = ( 'You are currently using the sandbox, so you ' 'can only grant qualifications that were ' 'also created in the sandbox.') else: sandbox_note = ( 'You are using the MTurk live site, so you ' 'can only grant qualifications that were ' 'also created on the live site, and not the ' 'MTurk sandbox.') msg = ( "In settings.py you specified qualification ID '{qualification_id}' " "MTurk returned the following error: [{exc}] " "Note: {sandbox_note}".format( qualification_id=qualification_id, exc=exc, sandbox_note=sandbox_note)) messages.error(request, msg) return HttpResponseRedirect( reverse('MTurkCreateHIT', args=(session.code, ))) hit = mturk_client.create_hit(**mturk_hit_parameters)['HIT'] session.mturk_HITId = hit['HITId'] session.mturk_HITGroupId = hit['HITGroupId'] session.mturk_use_sandbox = use_sandbox session.save() return HttpResponseRedirect( reverse('MTurkCreateHIT', args=(session.code, )))
if not netloc: raise OptionError("Unable to " "determine the hostname from " "the provided URL; please use a " "fully qualified URL.") scheme = scheme.lower() if scheme not in ("http", "https"): raise OptionError("Invalid URL; http " "and https are the only supported " "schemes.") # Rebuild the url with the sanitized components. ivalues["pkg"]["proxy_base"] = \ urlunparse((scheme, netloc, path, params, query, fragment)) elif opt == "--readonly": ivalues["pkg"]["readonly"] = True elif opt == "--rebuild": rebuild = True elif opt == "--refresh-index": # Note: This argument is for internal use # only. # # This flag is purposefully omitted in usage. # The supported way to forcefully reindex is to # kill any pkg.depot using that directory, # remove the index directory, and restart the # pkg.depot process. The index will be rebuilt # automatically on startup. reindex = True
def _create_endpoint(self, scheme, host, port, service_path): """Creates end point url for the service.""" netloc = '%s:%s' % (host, port) self._endpoint = urlparse.urlunparse( (scheme, netloc, service_path, None, None, None))
def base_url(self, filters, auth_data=None): """Base URL from catalog Filters can be: - service: compute, image, etc - region: the service region - endpoint_type: adminURL, publicURL, internalURL - api_version: replace catalog version with this - skip_path: take just the base URL """ if auth_data is None: auth_data = self.auth_data token, _auth_data = auth_data service = filters.get('service') region = filters.get('region') endpoint_type = filters.get('endpoint_type', 'public') if service is None: raise exceptions.EndpointNotFound("No service provided") if 'URL' in endpoint_type: endpoint_type = endpoint_type.replace('URL', '') _base_url = None catalog = _auth_data['catalog'] # Select entries with matching service type service_catalog = [ep for ep in catalog if ep['type'] == service] if len(service_catalog) > 0: service_catalog = service_catalog[0]['endpoints'] else: # No matching service raise exceptions.EndpointNotFound(service) # Filter by endpoint type (interface) filtered_catalog = [ ep for ep in service_catalog if ep['interface'] == endpoint_type ] if len(filtered_catalog) == 0: # No matching type, keep all and try matching by region at least filtered_catalog = service_catalog # Filter by region filtered_catalog = [ ep for ep in filtered_catalog if ep['region'] == region ] if len(filtered_catalog) == 0: # No matching region, take the first endpoint filtered_catalog = [service_catalog[0]] # There should be only one match. If not take the first. _base_url = filtered_catalog[0].get('url', None) if _base_url is None: raise exceptions.EndpointNotFound(service) parts = urlparse.urlparse(_base_url) if filters.get('api_version', None) is not None: version_path = '/%s' % filters['api_version'] path = re.sub(r'(^|/)+v\d+(?:\.\d+)?', version_path, parts.path, count=1) _base_url = urlparse.urlunparse( (parts.scheme, parts.netloc, path or version_path, parts.params, parts.query, parts.fragment)) if filters.get('skip_path', None) is not None: _base_url = urlparse.urlunparse( (parts.scheme, parts.netloc, '/', parts.params, parts.query, parts.fragment)) return _base_url
def addparams2url(url, params): if urlparse(url).query: return url + '&' + urlencode(params) else: return urlunparse(urlparse(url)) + '?' + urlencode(params)
def _DeriveCrmRegionalEndpoint(endpoint, location): scheme, netloc, path, params, query, fragment = parse.urlparse(endpoint) netloc = '{}-{}'.format(location, netloc) return parse.urlunparse((scheme, netloc, path, params, query, fragment))
def _open_url(self, url): """ Open an URL 'url' and return the file-like object of the opened URL. """ def _print_warning(timeout): """ This is a small helper function for printing a warning if we cannot open the URL for some time. """ _log.warning("failed to open the URL with %d sec timeout, is the " "proxy configured correctly? Keep trying ..." % timeout) import socket from six.moves import http_client as httplib from six.moves.urllib import request as urllib from six.moves.urllib.error import URLError parsed_url = urlparse.urlparse(url) if parsed_url.scheme == "ssh": # Unfortunately, urllib2 does not handle "ssh://" URLs self._open_url_ssh(parsed_url) return username = parsed_url.username password = parsed_url.password if username and password: # Unfortunately, in order to handle URLs which contain user name # and password (e.g., http://user:[email protected]), we need to # do few extra things. new_url = list(parsed_url) if parsed_url.port: new_url[1] = "%s:%s" % (parsed_url.hostname, parsed_url.port) else: new_url[1] = parsed_url.hostname url = urlparse.urlunparse(new_url) # Build an URL opener which will do the authentication password_manager = urllib.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, url, username, password) auth_handler = urllib.HTTPBasicAuthHandler(password_manager) opener = urllib.build_opener(auth_handler) else: opener = urllib.build_opener() opener.addheaders = [('User-Agent', 'Mozilla/5.0')] urllib.install_opener(opener) # Open the URL. First try with a short timeout, and print a message # which should supposedly give the a clue that something may be going # wrong. # The overall purpose of this is to improve user experience. For # example, if one tries to open a file but did not setup the proxy # environment variables propely, there will be a very long delay before # the failure message. And it is much nicer to pre-warn the user early # about something possibly being wrong. for timeout in (10, None): try: f_obj = opener.open(url, timeout=timeout) # Handling the timeout case in Python 2.7 except socket.timeout as err: if timeout is not None: _print_warning(timeout) else: raise Error("cannot open URL '%s': %s" % (url, err)) except URLError as err: # Handling the timeout case in Python 2.6 if timeout is not None and \ isinstance(err.reason, socket.timeout): _print_warning(timeout) else: raise Error("cannot open URL '%s': %s" % (url, err)) except (IOError, ValueError, httplib.InvalidURL) as err: raise Error("cannot open URL '%s': %s" % (url, err)) except httplib.BadStatusLine: raise Error("cannot open URL '%s': server responds with an " "HTTP status code that we don't understand" % url) self.is_url = True self._f_objs.append(f_obj)
def db_clean_legacy(): conf = cfg.ConfigOpts() conf.register_cli_opts([ cfg.strOpt('confirm-drop-table', short='n', help='confirm to drop the legacy tables') ]) if not conf.confirm_drop_table: confirm = moves.input("Do you really want to drop the legacy " "alarm and event tables? This will destroy " "data definitively if it exist. Please type " "'YES' to confirm: ") if confirm != 'YES': print("DB legacy cleanup aborted!") return service.prepare_service(conf=conf) url = (getattr(conf.database, "metering_connection") or conf.database.connection) parsed = urlparse.urlparse(url) if parsed.password: masked_netloc = '****'.join(parsed.netloc.rsplit(parsed.password)) masked_url = parsed._replace(netloc=masked_netloc) masked_url = urlparse.urlunparse(masked_url) else: masked_url = url LOG.info( _LI('Starting to drop event, alarm and alarm history tables in ' 'backend: %s'), masked_url) connection_scheme = parsed.scheme conn = storage.get_connection_from_config(conf) if connection_scheme in ('mysql', 'mysql+pymysql', 'postgresql', 'sqlite'): engine = conn._engine_facade.get_engine() meta = sa.MetaData(bind=engine) for table_name in ('alarm', 'alarm_history', 'trait_text', 'trait_int', 'trait_float', 'trait_datetime', 'event', 'event_type'): if engine.has_table(table_name): table = sa.Table(table_name, meta, autoload=True) table.drop() LOG.info( _LI("Legacy %s table of SQL backend has been " "dropped."), table_name) else: LOG.info(_LI('%s table does not exist.'), table_name) elif connection_scheme == 'hbase': with conn.conn_pool.connection() as h_conn: tables = h_conn.tables() table_name_mapping = { 'alarm': 'alarm', 'alarm_h': 'alarm history', 'event': 'event' } for table_name in ('alarm', 'alarm_h', 'event'): try: if table_name in tables: h_conn.disable_table(table_name) h_conn.delete_table(table_name) LOG.info( _LI("Legacy %s table of Hbase backend " "has been dropped."), table_name_mapping[table_name]) else: LOG.info(_LI('%s table does not exist.'), table_name_mapping[table_name]) except Exception as e: LOG.error( _LE('Error occurred while dropping alarm ' 'tables of Hbase, %s'), e) elif connection_scheme == 'mongodb': for table_name in ('alarm', 'alarm_history', 'event'): if table_name in conn.db.conn.collection_names(): conn.db.conn.drop_collection(table_name) LOG.info( _LI("Legacy %s table of Mongodb backend has been " "dropped."), table_name) else: LOG.info(_LI('%s table does not exist.'), table_name) LOG.info('Legacy alarm and event tables cleanup done.')
def _StartAction(self, mediaProvider): if not mediaProvider: raise RuntimeError('invalid mediaProvider') # if we are already connected check if something important changed in the media provider if self._connected: if kodi.Api.compareMediaProviders(self._mediaProvider, mediaProvider): # update the media provider and settings anyway self._mediaProvider = mediaProvider self._settings = self._mediaProvider.prepareSettings() return True self._StopAction(restart=True) self._mediaProvider = mediaProvider self._settings = self._mediaProvider.prepareSettings() if not self._settings: raise RuntimeError('cannot prepare media provider settings') try: # create emby server instance self._server = Server(self._mediaProvider) # authenticate with the Emby server authenticated = self._server.Authenticate() except: authenticated = False if not authenticated: ProviderObserver.log( 'failed to authenticate with {}'.format( mediaProvider2str(self._mediaProvider)), xbmc.LOGERROR) self._Reset() return False # analyze the media provider's URL urlParts = urlparse(self._mediaProvider.getBasePath()) # determine the proper scheme (ws:// or wss://) and whether or not to verify the HTTPS certificate websocketScheme = 'wss' if urlParts.scheme == 'https' else 'ws' # put the urL back together url = urlunparse( urlParts._replace(scheme=websocketScheme, path='embywebsocket')) url = Url.addOptions( url, { URL_QUERY_API_KEY: self._server.AccessToken(), URL_QUERY_DEVICE_ID: self._server.DeviceId() }) # create the websocket self._websocket = websocket.WebSocket() # connect the websocket try: self._websocket.connect(url) except Exception as err: ProviderObserver.log( 'failed to connect to {} using a websocket. {}'.format( url, err), xbmc.LOGERROR) self._Reset() return False # reduce the timeout self._websocket.settimeout(1.0) ProviderObserver.log( 'successfully connected to {} to observe media imports'.format( mediaProvider2str(self._mediaProvider))) self._connected = True return True
def get_canonicalized_asset_path(course_key, path, base_url, excluded_exts, encode=True): """ Returns a fully-qualified path to a piece of static content. If a static asset CDN is configured, this path will include it. Otherwise, the path will simply be relative. Args: course_key: key to the course which owns this asset path: the path to said content Returns: string: fully-qualified path to asset """ # Break down the input path. _, _, relative_path, params, query_string, _ = urlparse(path) # Convert our path to an asset key if it isn't one already. asset_key = StaticContent.get_asset_key_from_path( course_key, relative_path) # Check the status of the asset to see if this can be served via CDN aka publicly. serve_from_cdn = False content_digest = None try: content = AssetManager.find(asset_key, as_stream=True) serve_from_cdn = not getattr(content, "locked", True) content_digest = getattr(content, "content_digest", None) except (ItemNotFoundError, NotFoundError): # If we can't find the item, just treat it as if it's locked. serve_from_cdn = False # Do a generic check to see if anything about this asset disqualifies it from being CDN'd. is_excluded = False if StaticContent.is_excluded_asset_type(relative_path, excluded_exts): serve_from_cdn = False is_excluded = True # Update any query parameter values that have asset paths in them. This is for assets that # require their own after-the-fact values, like a Flash file that needs the path of a config # file passed to it e.g. /static/visualization.swf?configFile=/static/visualization.xml query_params = parse_qsl(query_string) updated_query_params = [] for query_name, query_val in query_params: if query_val.startswith("/static/"): new_val = StaticContent.get_canonicalized_asset_path( course_key, query_val, base_url, excluded_exts, encode=False) updated_query_params.append((query_name, new_val)) else: # Make sure we're encoding Unicode strings down to their byte string # representation so that `urlencode` can handle it. updated_query_params.append( (query_name, query_val.encode('utf-8'))) serialized_asset_key = StaticContent.serialize_asset_key_with_slash( asset_key) base_url = base_url if serve_from_cdn else '' asset_path = serialized_asset_key # If the content has a digest (i.e. md5sum) value specified, create a versioned path to the asset using it. if not is_excluded and content_digest: asset_path = StaticContent.add_version_to_asset_path( serialized_asset_key, content_digest) # Only encode this if told to. Important so that we don't double encode # when working with paths that are in query parameters. asset_path = asset_path.encode('utf-8') if encode: asset_path = quote_plus(asset_path, '/:+@') return urlunparse(('', base_url.encode('utf-8'), asset_path, params, urlencode(updated_query_params), ''))
def build_api_serving_url(spec_dict, origin_url=None, preferred_scheme=None): """The URL used to service API requests does not necessarily have to be the same URL that was used to retrieve the API spec_dict. The existence of three fields in the root of the specification govern the value of the api_serving_url: - host string The host (name or ip) serving the API. This MUST be the host only and does not include the scheme nor sub-paths. It MAY include a port. If the host is not included, the host serving the documentation is to be used (including the port). The host does not support path templating. - basePath string The base path on which the API is served, which is relative to the host. If it is not included, the API is served directly under the host. The value MUST start with a leading slash (/). The basePath does not support path templating. - schemes [string] The transfer protocol of the API. Values MUST be from the list: "http", "https", "ws", "wss". If the schemes is not included, the default scheme to be used is the one used to access the specification. See https://github.com/swagger-api/swagger-spec_dict/blob/master/versions/2.0.md#swagger-object- # noqa :param spec_dict: the Swagger spec in json-like dict form :param origin_url: the URL from which the spec was retrieved, if any. This is only used in Swagger clients. :param preferred_scheme: preferred scheme to use if more than one scheme is supported by the API. :return: base url which services api requests :raises: SwaggerSchemaError """ origin_url = origin_url or 'http://localhost/' origin = urlparse(origin_url) def pick_a_scheme(schemes): if not schemes: return origin.scheme if preferred_scheme: if preferred_scheme in schemes: return preferred_scheme raise SwaggerSchemaError( "Preferred scheme {0} not supported by API. Available schemes " "include {1}".format(preferred_scheme, schemes)) if origin.scheme in schemes: return origin.scheme if len(schemes) == 1: return schemes[0] raise SwaggerSchemaError( "Origin scheme {0} not supported by API. Available schemes " "include {1}".format(origin.scheme, schemes)) netloc = spec_dict.get('host', origin.netloc) path = spec_dict.get('basePath', '/') scheme = pick_a_scheme(spec_dict.get('schemes')) return urlunparse((scheme, netloc, path, None, None, None))
def get_repo_url(self): return urlunparse(("file", "", pathname2url(self.__dir), "", "", ""))
def encode_url_path(url, safe=SAFE_CHARS): from six.moves.urllib.parse import urlparse, urlunparse # pylint: disable=import-error url_parts = urlparse(url) quoted_path = encode_for_url(url_parts.path, safe) return urlunparse(url_parts[:2] + (quoted_path, ) + url_parts[3:])
def update_query_string(path, params, params_update_mode=QueryStringParamUpdateMode.REPLACE): ''' Update the query string parameters in a path. Parameters ---------- path: string The path to update parameters within. params: dict|list A dictionnary that contains keys and parameters to set in the query string params_update_mode: dict|string|list|int The default value is QueryStringParamUpdateMode.REPLACE that lead to replace value in the query string path by the one given in the params dictionary. It is possible to change the default behaviour giving the value QueryStringParamUpdateMode.APPEND. This will lead to always append values of the params dictionary to values of the query string path. The default behaviour can also be changed by specifying a parameter name as string, in this case only values for that parameter name will be appended. It can also contains a list or a tuple of parameter names for which values will be appended. Finally, this parameter can be a dictionary that specifies which parameter has to be appended or replaced. The dictionary contains parameter names in its keys and QueryStringParamUpdateMode in values. Returns ------- path: string The path updated with given parameters Example ------- A path containing a query string is:: /dir1/file1?param1=val1¶m2=val2¶mN=valN the params dictionary contains:: {'param1':'newval1', param2=newval2', param3':'newval3'} :: update_query_string('/dir1/file1?param1=val1¶m2=val2¶mN=valN', {'param1':'newval1', 'param2':'newval2', 'param3':'newval3'}) would return:: '/dir1/file1?param1=newval1¶m2=newval2¶mN=valN¶m3=newval3' :: update_query_string('/dir1/file1?param1=val1¶m2=val2¶mN=valN', {'param1':'newval1', 'param2':'newval2', 'param3':'newval3'}, QueryStringParamUpdateMode.APPEND) would return:: '/dir1/file1?param1=val1¶m1=newval1¶m2=val2¶m2=newval2¶mN=valN¶m3=newval3' :: update_query_string('/dir1/file1?param1=val1¶m2=val2¶mN=valN', {'param1':'newval1', 'param2':'newval2', 'param3':'newval3'}, 'param2') would return:: '/dir1/file1?param1=newval1¶m2=val2¶m2=newval2¶mN=valN¶m3=newval3' :: update_query_string('/dir1/file1?param1=val1¶m2=val2¶mN=valN', {'param1':'newval1', 'param2':'newval2', 'param3':'newval3'}, ('param1', 'param2')) would return:: '/dir1/file1?param1=val1¶m1=newval1¶m2=val2¶m2=newval2¶mN=valN¶m3=newval3' :: update_query_string('/dir1/file1?param1=val1¶m2=val2¶mN=valN', {'param1':'newval1', 'param2':'newval2', 'param3':'newval3'}, {'param1': QueryStringParamUpdateMode.APPEND, 'param2': QueryStringParamUpdateMode.REPLACE}) would return:: '/dir1/file1?param1=val1¶m1=newval1¶m2=val2¶m2=newval2¶mN=valN¶m3=newval3' ''' from six.moves.urllib import parse as urllib urlparse = urllib # Convert params_update_mode to a dictionary that contains the update mode # for each parameter if type(params_update_mode) in (list, tuple): # Update mode is specified using a list of parameter names default_update_mode = QueryStringParamUpdateMode.REPLACE params_update = params_update_mode params_update_mode = dict() for p in params_update: if (type(p) in (list, tuple)): if (len(p) > 1): params_update_mode[p[0]] = p[1] elif (len(p) > 0): params_update_mode[ p[0]] = QueryStringParamUpdateMode.APPEND else: params_update_mode[p] = QueryStringParamUpdateMode.APPEND elif isinstance(params_update_mode, six.string_types): # A parameter name was given directly default_update_mode = QueryStringParamUpdateMode.REPLACE params_update_mode = dict( ((params_update_mode, QueryStringParamUpdateMode.APPEND), )) elif params_update_mode in (QueryStringParamUpdateMode.APPEND, QueryStringParamUpdateMode.REPLACE, QueryStringParamUpdateMode.REMOVE): # Update mode was specified for all parameters default_update_mode = params_update_mode params_update_mode = dict() elif type(params_update_mode) is dict: default_update_mode = QueryStringParamUpdateMode.REPLACE else: raise RuntimeError( 'params_update_mode is not specified correctly. ' 'It must be either a dictionary that contains parameter names ' 'and the corresponding QueryStringParamUpdateMode, ' 'either a list that contains parameter names, either' 'QueryStringParamUpdateMode.') url_parsed = strict_urlparse(path) url_params = urlparse.parse_qs(url_parsed.query) if isinstance(params, (list, tuple)): params = dict([(p, '') for p in params]) # Update parameters dictionary for p, v in six.iteritems(params): update_mode = params_update_mode.get(p, default_update_mode) if update_mode == QueryStringParamUpdateMode.REPLACE: url_params[p] = v elif update_mode == QueryStringParamUpdateMode.APPEND: if type(v) in (list, tuple): if type(v) is tuple: url_params[p] += list(v) else: url_params[p] += v else: url_params.setdefault(p, list()).append(v) elif update_mode == QueryStringParamUpdateMode.REMOVE: del url_params[p] else: raise RuntimeError( 'params_update_mode is not specified correctly. %s is ' 'not a valid value for parameter %s. Valid values are ' 'either QueryStringParamUpdateMode.APPEND, either' 'QueryStringParamUpdateMode.REPLACE.' % (v, p)) url_new = list(url_parsed) url_new[4] = urllib.urlencode(url_params, doseq=True) return urlparse.urlunparse(url_new)
def url(tgpath, tgparams=None, **kwargs): '''Computes URLs. This is a replacement for :func:`turbogears.controllers.url` (aka :func:`tg.url` in the template). In addition to the functionality that :func:`tg.url` provides, it adds a token to prevent :term:`CSRF` attacks. :arg tgpath: a list or a string. If the path is absolute (starts with a "/"), the :attr:`server.webpath`, :envvar:`SCRIPT_NAME` and the approot of the application are prepended to the path. In order for the approot to be detected properly, the root object should extend :class:`turbogears.controllers.RootController`. :kwarg tgparams: See param: ``kwargs`` :kwarg kwargs: Query parameters for the URL can be passed in as a dictionary in the second argument *or* as keyword parameters. Values which are a list or a tuple are used to create multiple key-value pairs. :returns: The changed path .. versionadded:: 0.3.10 Modified from turbogears.controllers.url for :ref:`CSRF-Protection` ''' if not isinstance(tgpath, six.string_types): tgpath = '/'.join(list(tgpath)) if tgpath.startswith('/'): webpath = (config.get('server.webpath') or '').rstrip('/') if tg_util.request_available(): check_app_root() tgpath = request.app_root + tgpath try: webpath += request.wsgi_environ['SCRIPT_NAME'].rstrip('/') except (AttributeError, KeyError): # pylint: disable-msg=W0704 # :W0704: Lack of wsgi environ is fine... we still have # server.webpath pass tgpath = webpath + tgpath if tgparams is None: tgparams = kwargs else: try: tgparams = tgparams.copy() tgparams.update(kwargs) except AttributeError: raise TypeError('url() expects a dictionary for query parameters') args = [] # Add the _csrf_token try: if identity.current.csrf_token: tgparams.update({'_csrf_token': identity.current.csrf_token}) except RequestRequiredException: # pylint: disable-msg=W0704 # :W0704: If we are outside of a request (called from non-controller # methods/ templates) just don't set the _csrf_token. pass # Check for query params in the current url query_params = six.iteritems(tgparams) scheme, netloc, path, params, query_s, fragment = urlparse(tgpath) if query_s: query_params = chain( (p for p in cgi.parse_qsl(query_s) if p[0] != '_csrf_token'), query_params) for key, value in query_params: if value is None: continue if isinstance(value, (list, tuple)): pairs = [(key, v) for v in value] else: pairs = [(key, value)] for key, value in pairs: if value is None: continue if isinstance(value, unicode): value = value.encode('utf8') args.append((key, str(value))) query_string = urlencode(args, True) tgpath = urlunparse((scheme, netloc, path, params, query_string, fragment)) return tgpath
def test_is_valid_url(url): unparsed_url = urllib_parse.urlunparse(url) assume(not unparsed_url.startswith("file://")) assert vistir.path.is_valid_url(unparsed_url)
def get_logged_in_program_certificate_url(certificate_url): parsed_url = urlparse(certificate_url) query_string = 'next=' + parsed_url.path url_parts = (parsed_url.scheme, parsed_url.netloc, '/login/', '', query_string, '') return urlunparse(url_parts)
def prepare_service(args=None, conf=None, default_config_files=None, log_to_std=False, logging_level=None): if conf is None: conf = cfg.ConfigOpts() opts.set_defaults() # FIXME(jd) Use the pkg_entry info to register the options of these libs db_options.set_defaults(conf) policy_opts.set_defaults(conf) # Register our own Gnocchi options for group, options in opts.list_opts(): conf.register_opts(list(options), group=None if group == "DEFAULT" else group) conf.register_cli_opts(opts._cli_options) conf.set_default("workers", utils.get_default_workers(), group="metricd") conf(args, project='gnocchi', validate_default_values=True, default_config_files=default_config_files, version=pbr.version.VersionInfo('gnocchi').version_string()) if not log_to_std and (conf.log_dir or conf.log_file): outputs = [daiquiri.output.File(filename=conf.log_file, directory=conf.log_dir)] else: outputs = [daiquiri.output.STDERR] if conf.use_syslog: outputs.append( daiquiri.output.Syslog(facility=conf.syslog_log_facility)) if conf.use_journal: outputs.append(daiquiri.output.Journal()) daiquiri.setup(outputs=outputs) if logging_level is None: if conf.debug: logging_level = logging.DEBUG elif conf.verbose: logging_level = logging.INFO else: logging_level = logging.WARNING logging.getLogger("gnocchi").setLevel(logging_level) # HACK(jd) I'm not happy about that, fix AP class to handle a conf object? archive_policy.ArchivePolicy.DEFAULT_AGGREGATION_METHODS = ( conf.archive_policy.default_aggregation_methods ) # If no coordination URL is provided, default to using the indexer as # coordinator if conf.storage.coordination_url is None: if conf.storage.driver == "redis": conf.set_default("coordination_url", conf.storage.redis_url, "storage") elif conf.incoming.driver == "redis": conf.set_default("coordination_url", conf.incoming.redis_url, "storage") else: parsed = urlparse.urlparse(conf.indexer.url) proto, _, _ = parsed.scheme.partition("+") parsed = list(parsed) # Set proto without the + part parsed[0] = proto conf.set_default("coordination_url", urlparse.urlunparse(parsed), "storage") cfg_path = conf.oslo_policy.policy_file if not os.path.isabs(cfg_path): cfg_path = conf.find_file(cfg_path) if cfg_path is None or not os.path.exists(cfg_path): cfg_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'rest', 'policy.json')) conf.set_default('policy_file', cfg_path, group='oslo_policy') conf.log_opt_values(LOG, logging.DEBUG) return conf
def quote_url(url): parsed = urlparse(url) if parsed.scheme not in ('http', 'https'): return url parsed = parsed._replace(path=quote(parsed.path)) return urlunparse(parsed)
def replacer(siteurl, m): what = m.group('what') value = urlparse(m.group('value')) path = value.path origin = m.group('path') # XXX Put this in a different location. if what in {'filename', 'attach'}: if path.startswith('/'): path = path[1:] else: # relative to the source path of this content path = content.get_relative_source_path( os.path.join(content.relative_dir, path)) if path not in content._context['filenames']: unquoted_path = path.replace('%20', ' ') if unquoted_path in content._context['filenames']: path = unquoted_path linked_content = content._context['filenames'].get(path) if linked_content: if what == 'attach': if isinstance(linked_content, Static): linked_content.attach_to(content) else: logger.warning( "%s used {attach} link syntax on a " "non-static file. Use {filename} instead.", content.get_relative_source_path()) origin = '/'.join((siteurl, linked_content.url)) origin = origin.replace('\\', '/') # for Windows paths. else: logger.warning( "Unable to find `%s`, skipping url replacement.", value.geturl(), extra={ 'limit_msg': ("Other resources were not found " "and their urls not replaced") }) elif what == 'category': origin = '/'.join((siteurl, Category(path, content.settings).url)) elif what == 'tag': origin = '/'.join((siteurl, Tag(path, content.settings).url)) elif what == 'index': origin = '/'.join((siteurl, content.settings['INDEX_SAVE_AS'])) elif what == 'author': origin = '/'.join((siteurl, Author(path, content.settings).url)) else: logger.warning( "Replacement Indicator '%s' not recognized, " "skipping replacement", what) # keep all other parts, such as query, fragment, etc. parts = list(value) parts[2] = origin origin = urlunparse(parts) return ''.join( (m.group('markup'), m.group('quote'), origin, m.group('quote')))
def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, encoding=None): r"""Canonicalize the given url by applying the following procedures: - sort query arguments, first by key, then by value - percent encode paths ; non-ASCII characters are percent-encoded using UTF-8 (RFC-3986) - percent encode query arguments ; non-ASCII characters are percent-encoded using passed `encoding` (UTF-8 by default) - normalize all spaces (in query arguments) '+' (plus symbol) - normalize percent encodings case (%2f -> %2F) - remove query arguments with blank values (unless `keep_blank_values` is True) - remove fragments (unless `keep_fragments` is True) The url passed can be bytes or unicode, while the url returned is always a native str (bytes in Python 2, unicode in Python 3). >>> import w3lib.url >>> >>> # sorting query arguments >>> w3lib.url.canonicalize_url('http://www.example.com/do?c=3&b=5&b=2&a=50') 'http://www.example.com/do?a=50&b=2&b=5&c=3' >>> >>> # UTF-8 conversion + percent-encoding of non-ASCII characters >>> w3lib.url.canonicalize_url(u'http://www.example.com/r\u00e9sum\u00e9') 'http://www.example.com/r%C3%A9sum%C3%A9' >>> For more examples, see the tests in `tests/test_url.py`. """ # If supplied `encoding` is not compatible with all characters in `url`, # fallback to UTF-8 as safety net. # UTF-8 can handle all Unicode characters, # so we should be covered regarding URL normalization, # if not for proper URL expected by remote website. try: scheme, netloc, path, params, query, fragment = _safe_ParseResult( parse_url(url), encoding=encoding) except UnicodeEncodeError as e: scheme, netloc, path, params, query, fragment = _safe_ParseResult( parse_url(url), encoding='utf8') # 1. decode query-string as UTF-8 (or keep raw bytes), # sort values, # and percent-encode them back if six.PY2: keyvals = parse_qsl(query, keep_blank_values) else: # Python3's urllib.parse.parse_qsl does not work as wanted # for percent-encoded characters that do not match passed encoding, # they get lost. # # e.g., 'q=b%a3' becomes [('q', 'b\ufffd')] # (ie. with 'REPLACEMENT CHARACTER' (U+FFFD), # instead of \xa3 that you get with Python2's parse_qsl) # # what we want here is to keep raw bytes, and percent encode them # so as to preserve whatever encoding what originally used. # # See https://tools.ietf.org/html/rfc3987#section-6.4: # # For example, it is possible to have a URI reference of # "http://www.example.org/r%E9sum%E9.xml#r%C3%A9sum%C3%A9", where the # document name is encoded in iso-8859-1 based on server settings, but # where the fragment identifier is encoded in UTF-8 according to # [XPointer]. The IRI corresponding to the above URI would be (in XML # notation) # "http://www.example.org/r%E9sum%E9.xml#résumé". # Similar considerations apply to query parts. The functionality of # IRIs (namely, to be able to include non-ASCII characters) can only be # used if the query part is encoded in UTF-8. keyvals = parse_qsl_to_bytes(query, keep_blank_values) keyvals.sort() query = urlencode(keyvals) # 2. decode percent-encoded sequences in path as UTF-8 (or keep raw bytes) # and percent-encode path again (this normalizes to upper-case %XX) uqp = _unquotepath(path) path = quote(uqp, _safe_chars) or '/' fragment = '' if not keep_fragments else fragment # every part should be safe already return urlunparse( (scheme, netloc.lower().rstrip(':'), path, params, query, fragment))
def _load(self, path, crawl=True, originaluri=None, includelogs=False,\ init=True, loadtype='href', loadcomplete=False, \ rel=False, prevpath=None): """Helper function to main load function. :param path: path to start load from. :type path: str. :param crawl: flag to determine if load should traverse found links. :type crawl: boolean. :param originaluri: variable to assist in determining originating path. :type originaluri: str. :param includelogs: flag to determine if logs should be downloaded also. :type includelogs: boolean. :param init: flag to determine if first run of load. :type init: boolean. :param loadtype: flag to determine if load is meant for only href items. :type loadtype: str. :param loadcomplete: flag to download the entire monolith :type loadcomplete: boolean """ if path.endswith("?page=1") and not loadcomplete: return elif not includelogs and not crawl: if "/Logs" in path: return #TODO: need to find a better way to support non ascii characters path = path.replace("|", "%7C") #remove fragments newpath = urlparse(path) newpath = list(newpath[:]) newpath[-1] = '' path = urlunparse(tuple(newpath)) if prevpath and prevpath != path: self.ctree[prevpath].update([path]) if not rel: if path.lower() in self.visited_urls: return LOGGER.debug('_loading %s', path) resp = self._client().get(path) if resp.status != 200 and path.lower() == self.typepath.defs.biospath: raise BiosUnregisteredError() elif resp.status == 401: raise SessionExpired("Invalid session. Please logout and "\ "log back in or include credentials.") elif resp.status not in (201, 200): self.removepath(path) return if loadtype == "ref": try: if resp.status in (201, 200): self.update_member(resp=resp, path=path, init=init) self.parse_schema(resp) except jsonpointer.JsonPointerException: raise SchemaValidationError() self.update_member(resp=resp, path=path, init=init) fpath = lambda pa, path: path if pa.endswith(self.typepath.defs.hrefstring) and \ pa.startswith((self.collstr, 'Entries')) else None if loadtype == 'href': #follow all the href attributes if self.is_redfish: jsonpath_expr = jsonpath_rw.parse("$..'@odata.id'") else: jsonpath_expr = jsonpath_rw.parse('$..href') matches = jsonpath_expr.find(resp.dict) if 'links' in resp.dict and 'NextPage' in resp.dict['links']: if originaluri: next_link_uri = originaluri + '?page=' + \ str(resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=originaluri, \ includelogs=includelogs, crawl=crawl, \ init=init, prevpath=None, loadcomplete=loadcomplete) else: next_link_uri = path + '?page=' + \ str(resp.dict['links']['NextPage']['page']) href = '%s' % next_link_uri self._load(href, originaluri=path, includelogs=includelogs,\ crawl=crawl, init=init, prevpath=None, loadcomplete=loadcomplete) matchrdirpath = next((match for match in matches if match.value == \ self._resourcedir), None) if not matchrdirpath and crawl: for match in matches: if path == "/rest/v1" and not loadcomplete: if str(match.full_path) == "links.Schemas.href" or \ str(match.full_path) == "links.Registries.href": continue elif not loadcomplete: if str(match.full_path) == "*****@*****.**" or \ str(match.full_path) == "*****@*****.**": continue if match.value == path: continue elif not isinstance(match.value, six.string_types): continue href = '%s' % match.value self._load(href, crawl=crawl, \ originaluri=originaluri, includelogs=includelogs, \ init=init, prevpath=fpath(str(match.full_path), path), \ loadcomplete=loadcomplete) elif crawl: href = '%s' % matchrdirpath.value self._load(href, crawl=crawl, originaluri=originaluri, \ includelogs=includelogs, init=init, prevpath=path, loadcomplete=loadcomplete) if loadcomplete: if path == '/rest/v1': schemamatch = jsonpath_rw.parse('$..extref') else: schemamatch = jsonpath_rw.parse('$..Uri') smatches = schemamatch.find(resp.dict) matches = matches + smatches for match in matches: if isinstance(match.value, six.string_types): self._load(match.value, crawl=crawl, originaluri=originaluri,\ includelogs=includelogs, init=init, loadcomplete=loadcomplete,\ prevpath=fpath(str(match.full_path), path))
def DeriveRegionalEndpoint(endpoint, region): scheme, netloc, path, params, query, fragment = urlparse.urlparse(endpoint) netloc = '{}-{}'.format(region, netloc) return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
def geturl(self): return urlunparse((self.scheme, self.netloc, self.path, self.params, self.query, self.fragment))
def _marshal_uri(target_uri, origin_uri): """ Translate the URL string representation into a new string which could be used as JSON keys. This method is needed because many JSON parsers and reference resolvers are using '/' as indicator of object nesting. To workaround this limitation we can re-write the url representation in a way that the parsers will accept it, for example "#/definitions/data_type" could become "|..definitions..data_type" Example: Assume that you have the following JSON document { "definitions": { "a/possible/def": { "type": "object" }, "a": { "possible": { "def": { "type": "string" } } }, "def": { "$ref": "#/definitions/a/possible/def" } } } Assuming that the JSON parser is not raising exception the dereferenced value of "#/definitions/def" could be {"type": "object"} or {"type": "string"} which is an undetermined condition which can lead to weird errors. Let's assume instead that the JSON parser will raise an exception in this case the JSON object will not be usable. To prevent this conditions we are removing possible '/' from the JSON keys. :param target_uri: URI to marshal :type target_uri: ParseResult :param origin_uri: URI of the root swagger spec file :type origin_uri: ParseResult :return: a string representation of the URL which could be used into the JSON keys :rtype: str """ marshalled_target = urlunparse(target_uri) if marshalled_target and target_uri.scheme == '': # scheme is empty for relative paths. It should NOT happen! target_uri = ParseResult('file', *target_uri[1:]) marshalled_target = urlunparse(target_uri) if not marshalled_target or target_uri.scheme not in { 'file', 'http', 'https' }: raise ValueError('Invalid target: \'{target_uri}\''.format( target_uri=urlunparse(target_uri))) if origin_uri and target_uri.scheme == 'file': scheme, netloc, path, params, query, fragment = target_uri # Masquerade the absolute file path on the "local" server using # relative paths from the root swagger spec file spec_dir = os.path.dirname(origin_uri.path) scheme = 'lfile' path = os.path.relpath(path, spec_dir) marshalled_target = urlunparse( (scheme, netloc, path, params, query, fragment)) for src, dst in iteritems(MARSHAL_REPLACEMENT_PATTERNS): marshalled_target = marshalled_target.replace(src, dst) return marshalled_target
import pytest from cloudmanager.setup_db import setup_db import psycopg2 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from settings import TEST_DB_PATH #from utils.migrate import run_migrations test_uri = list(urlparse(TEST_DB_PATH)) DB_NAME = test_uri[2].replace('/', '') # Connection URI used for creating the test database # Strip the database name from the URI since the DB doesn't exist yet test_uri[2] = '' PG_URI = urlunparse(test_uri) DB_CONN = None # Connection for creating the test database DB_TEST_CONN = None # Connection to the test database used by tests def pytest_configure(config): """Create test database and initialize connection""" global DB_CONN, DB_TEST_CONN try: DB_CONN = psycopg2.connect(PG_URI) DB_CONN.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) cur = DB_CONN.cursor() try: cur.execute('CREATE DATABASE %s;' % DB_NAME) except: cur.execute('DROP DATABASE %s;' % DB_NAME)
def _get_unversioned_endpoint(base_url): endpoint_parts = urlparse.urlparse(base_url) new_path = _remove_version_project(endpoint_parts.path) endpoint_parts = endpoint_parts._replace(path=new_path) endpoint = urlparse.urlunparse(endpoint_parts) return endpoint
def _finalize_core(self, **defaults): """ Complete initialization of standard IPA environment. This method will perform the following steps: 1. Call `Env._bootstrap()` if it hasn't already been called. 2. Merge-in variables from the configuration file ``self.conf`` (if it exists) by calling `Env._merge_from_file()`. 3. Merge-in variables from the defaults configuration file ``self.conf_default`` (if it exists) by calling `Env._merge_from_file()`. 4. Intelligently fill-in the *in_server* , *logdir*, *log*, and *jsonrpc_uri* variables if they haven't already been set. 5. Merge-in the variables in ``defaults`` by calling `Env._merge()`. In normal circumstances ``defaults`` will simply be those specified in `constants.DEFAULT_CONFIG`. After this method is called, all the environment variables used by all the built-in plugins will be available. As such, this method should be called *before* any plugins are loaded. After this method has finished, the `Env` instance is still writable so that 3rd-party plugins can set variables they may require as the plugins are registered. Also see `Env._finalize()`, the final method in the bootstrap sequence. :param defaults: Internal defaults for all built-in variables. """ self.__doing('_finalize_core') self.__do_if_not_done('_bootstrap') # Merge in context config file and then default config file: if self.__d.get('mode', None) != 'dummy': self._merge_from_file(self.conf) self._merge_from_file(self.conf_default) # Determine if in_server: if 'in_server' not in self: self.in_server = (self.context == 'server') # Set logdir: if 'logdir' not in self: if self.in_tree or not self.in_server: self.logdir = self._join('dot_ipa', 'log') else: self.logdir = path.join('/', 'var', 'log', 'ipa') # Set log file: if 'log' not in self: self.log = self._join('logdir', '%s.log' % self.context) # Workaround for ipa-server-install --uninstall. When no config file # is available, we set realm, domain, and basedn to RFC 2606 reserved # suffix to suppress attribute errors during uninstallation. if (self.in_server and self.context == 'installer' and not getattr(self, 'config_loaded', False)): if 'realm' not in self: self.realm = 'UNCONFIGURED.INVALID' if 'domain' not in self: self.domain = self.realm.lower() if 'basedn' not in self and 'domain' in self: self.basedn = DN(*(('dc', dc) for dc in self.domain.split('.'))) # Derive xmlrpc_uri from server # (Note that this is done before deriving jsonrpc_uri from xmlrpc_uri # and server from jsonrpc_uri so that when only server or xmlrpc_uri # is specified, all 3 keys have a value.) if 'xmlrpc_uri' not in self and 'server' in self: self.xmlrpc_uri = 'https://{}/ipa/xml'.format(self.server) # Derive ldap_uri from server if 'ldap_uri' not in self and 'server' in self: self.ldap_uri = 'ldap://{}'.format(self.server) # Derive jsonrpc_uri from xmlrpc_uri if 'jsonrpc_uri' not in self: if 'xmlrpc_uri' in self: xmlrpc_uri = self.xmlrpc_uri else: xmlrpc_uri = defaults.get('xmlrpc_uri') if xmlrpc_uri: (scheme, netloc, uripath, params, query, fragment ) = urlparse(xmlrpc_uri) uripath = uripath.replace('/xml', '/json', 1) self.jsonrpc_uri = urlunparse(( scheme, netloc, uripath, params, query, fragment)) if 'server' not in self: if 'jsonrpc_uri' in self: jsonrpc_uri = self.jsonrpc_uri else: jsonrpc_uri = defaults.get('jsonrpc_uri') if jsonrpc_uri: parsed = urlparse(jsonrpc_uri) self.server = parsed.netloc self._merge(**defaults) # set the best known TLS version if min/max versions are not set if 'tls_version_min' not in self: self.tls_version_min = TLS_VERSIONS[-1] elif self.tls_version_min not in TLS_VERSIONS: raise errors.EnvironmentError( "Unknown TLS version '{ver}' set in tls_version_min." .format(ver=self.tls_version_min)) if 'tls_version_max' not in self: self.tls_version_max = TLS_VERSIONS[-1] elif self.tls_version_max not in TLS_VERSIONS: raise errors.EnvironmentError( "Unknown TLS version '{ver}' set in tls_version_max." .format(ver=self.tls_version_max)) if self.tls_version_max < self.tls_version_min: raise errors.EnvironmentError( "tls_version_min is set to a higher TLS version than " "tls_version_max.")