def get_url(self): """Return the base URL of the Foreman deployment being tested. The following values from the config file are used to build the URL: * ``[server] scheme`` (default: https) * ``[server] hostname`` (required) * ``[server] port`` (default: none) Setting ``port`` to 80 does *not* imply that ``scheme`` is 'https'. If ``port`` is 80 and ``scheme`` is unset, ``scheme`` will still default to 'https'. :return: A URL. :rtype: str """ if not self.scheme: scheme = "https" else: scheme = self.scheme # All anticipated error cases have been handled at this point. if not self.port: return urlunsplit((scheme, self.hostname, "", "", "")) else: return urlunsplit((scheme, "{0}:{1}".format(self.hostname, self.port), "", "", ""))
def __init__(self, address, name=""): """Bind the publisher class to a port. """ # pylint: disable=E1103 self.name = name self.destination = address self.publish = get_context().socket(zmq.PUB) # Check for port 0 (random port) u__ = urlsplit(self.destination) port = u__.port if port == 0: dest = urlunsplit((u__.scheme, u__.hostname, u__.path, u__.query, u__.fragment)) self.port_number = self.publish.bind_to_random_port(dest) netloc = u__.hostname + ":" + str(self.port_number) self.destination = urlunsplit((u__.scheme, netloc, u__.path, u__.query, u__.fragment)) else: self.publish.bind(self.destination) self.port_number = port LOGGER.info("publisher started on port %s", str(self.port_number)) # Initialize no heartbeat self._heartbeat = None self._pub_lock = Lock()
def __init__(self, url, application=None, session=None, output_grid=True): # download DDS/DAS scheme, netloc, path, query, fragment = urlsplit(url) ddsurl = urlunsplit((scheme, netloc, path + ".dds", query, fragment)) r = GET(ddsurl, application, session) raise_for_status(r) dds = r.text dasurl = urlunsplit((scheme, netloc, path + ".das", query, fragment)) r = GET(dasurl, application, session) raise_for_status(r) das = r.text # build the dataset from the DDS and add attributes from the DAS self.dataset = build_dataset(dds) add_attributes(self.dataset, parse_das(das)) # remove any projection from the url, leaving selections projection, selection = parse_ce(query) url = urlunsplit((scheme, netloc, path, "&".join(selection), fragment)) # now add data proxies for var in walk(self.dataset, BaseType): var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session) for var in walk(self.dataset, SequenceType): template = copy.copy(var) var.data = SequenceProxy(url, template, application=application, session=session) # apply projections for var in projection: target = self.dataset while var: token, index = var.pop(0) target = target[token] if isinstance(target, BaseType): target.data.slice = fix_slice(index, target.shape) elif isinstance(target, GridType): index = fix_slice(index, target.array.shape) target.array.data.slice = index for s, child in zip(index, target.maps): target[child].data.slice = (s,) elif isinstance(target, SequenceType): target.data.slice = index # retrieve only main variable for grid types: for var in walk(self.dataset, GridType): var.set_output_grid(output_grid)
def remove_version_from_href(href): """Removes the first api version from the href. Given: 'http://www.nova.com/v1.1/123' Returns: 'http://www.nova.com/123' Given: 'http://www.nova.com/v1.1' Returns: 'http://www.nova.com' """ parsed_url = urlparse.urlsplit(href) url_parts = parsed_url.path.split('/', 2) # NOTE: this should match vX.X or vX expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)') if expression.match(url_parts[1]): del url_parts[1] new_path = '/'.join(url_parts) if new_path == parsed_url.path: LOG.debug('href %s does not contain version' % href) raise ValueError(_('href %s does not contain version') % href) parsed_url = list(parsed_url) parsed_url[2] = new_path return urlparse.urlunsplit(parsed_url)
def _get_safe_url(url): """Gets version of *url* with basic auth passwords obscured. This function returns results suitable for printing and logging. E.g.: https://user:[email protected] => https://user:********@example.com .. note:: The number of astrisks is invariant in the length of the basic auth password, so minimal information is leaked. :param url: a url :type url: ``str`` :return: *url* with password obscured :rtype: ``str`` """ safe_url = url url, username, _ = _strip_basic_auth(url) if username is not None: # case: url contained basic auth creds; obscure password url_parts = parse.urlsplit(url) safe_netloc = '{0}@{1}'.format(username, url_parts.hostname) # replace original netloc w/ obscured version frags = list(url_parts) frags[1] = safe_netloc safe_url = parse.urlunsplit(frags) return safe_url
def get_authorization_url(self, redirect_url): """ Get the authorization url based on the client id and the redirect url passed in :param redirect_url: An HTTPS URI or custom URL scheme where the response will be redirected. Optional if the redirect URI is registered with Box already. :type redirect_url: `unicode` or None :return: A tuple of the URL of Box's authorization page and the CSRF token. This is the URL that your application should forward the user to in first leg of OAuth 2. :rtype: (`unicode`, `unicode`) """ csrf_token = self._get_state_csrf_token() # For the query string parameters, use a sequence of two-element # tuples, rather than a dictionary, in order to get a consistent and # predictable order of parameters in the output of `urlencode()`. params = [ ('state', csrf_token), ('response_type', 'code'), ('client_id', self._client_id), ] if redirect_url: params.append(('redirect_uri', redirect_url)) # `urlencode()` doesn't work with non-ASCII unicode characters, so # encode the parameters as ASCII bytes. params = [(key.encode('utf-8'), value.encode('utf-8')) for (key, value) in params] query_string = urlencode(params) return urlunsplit(('', '', API.OAUTH2_AUTHORIZE_URL, query_string, '')), csrf_token
def _proxy_request(self, instance_id, tenant_id, req): headers = { 'X-Forwarded-For': req.headers.get('X-Forwarded-For'), 'X-Instance-ID': instance_id, 'X-Tenant-ID': tenant_id, 'X-Instance-ID-Signature': self._sign_instance_id(instance_id) } nova_host_port = '%s:%s' % (self.conf.nova_metadata_host, self.conf.nova_metadata_port) url = urlparse.urlunsplit(( self.conf.nova_metadata_protocol, nova_host_port, req.path_info, req.query_string, '')) disable_ssl_certificate_validation = self.conf.nova_metadata_insecure if self.conf.auth_ca_cert and not disable_ssl_certificate_validation: verify_cert = self.conf.auth_ca_cert else: verify_cert = not disable_ssl_certificate_validation client_cert = None if self.conf.nova_client_cert and self.conf.nova_client_priv_key: client_cert = (self.conf.nova_client_cert, self.conf.nova_client_priv_key) resp = requests.request(method=req.method, url=url, headers=headers, data=req.body, cert=client_cert, verify=verify_cert) if resp.status_code == 200: req.response.content_type = resp.headers['content-type'] req.response.body = resp.content LOG.debug(str(resp)) return req.response elif resp.status_code == 403: LOG.warning( 'The remote metadata server responded with Forbidden. This ' 'response usually occurs when shared secrets do not match.' ) return webob.exc.HTTPForbidden() elif resp.status_code == 400: return webob.exc.HTTPBadRequest() elif resp.status_code == 404: return webob.exc.HTTPNotFound() elif resp.status_code == 409: return webob.exc.HTTPConflict() elif resp.status_code == 500: msg = _( 'Remote metadata server experienced an internal server error.' ) LOG.warning(msg) explanation = six.text_type(msg) return webob.exc.HTTPInternalServerError(explanation=explanation) else: raise Exception(_('Unexpected response code: %s') % resp.status)
def set_url(self, url): path = url.lstrip('/') if url.startswith("http://") or url.startswith("https://"): u = urlparse.urlsplit(url) if u.username is not None: password = u.password or "" encode = base64.b64encode("%s:%s" % (u.username, password)) self.headers['Authorization'] = 'Basic %s' % encode self.scheme = u.scheme, self.host = u.netloc.split("@")[-1] self.path_info = u.path or "/" self.query_string = u.query url = urlparse.urlunsplit((u.scheme, u.netloc.split("@")[-1], u.path, u.query, u.fragment)) else: if '?' in path: path, self.query_string = path.split('?', 1) self.path_info = '/' + path url = self.url self.scheme, self.host, self.path_info = urlparse.urlparse(url)[0:3]
def download_attachments(output_path, urls): """Downloads WordPress attachments and returns a list of paths to attachments that can be associated with a post (relative path to output directory). Files that fail to download, will not be added to posts""" locations = {} for url in urls: path = urlparse(url).path # teardown path and rebuild to negate any errors with # os.path.join and leading /'s path = path.split('/') filename = path.pop(-1) localpath = '' for item in path: if sys.platform != 'win32' or ':' not in item: localpath = os.path.join(localpath, item) full_path = os.path.join(output_path, localpath) # Generate percent-encoded URL scheme, netloc, path, query, fragment = urlsplit(url) path = quote(path) url = urlunsplit((scheme, netloc, path, query, fragment)) if not os.path.exists(full_path): os.makedirs(full_path) print('downloading {}'.format(filename)) try: urlretrieve(url, os.path.join(full_path, filename)) locations[url] = os.path.join(localpath, filename) except (URLError, IOError) as e: # Python 2.7 throws an IOError rather Than URLError logger.warning("No file could be downloaded from %s\n%s", url, e) return locations
def authenticate(self): magic_tuple = netutils.urlsplit(self.auth_url) scheme, netloc, path, query, frag = magic_tuple port = magic_tuple.port if port is None: port = 80 path_parts = path.split('/') for part in path_parts: if len(part) > 0 and part[0] == 'v': self.version = part break if self.auth_token and self.management_url: self._save_keys() return # TODO(sandy): Assume admin endpoint is 35357 for now. # Ideally this is going to have to be provided by the service catalog. new_netloc = netloc.replace(':%d' % port, ':%d' % (35357,)) admin_url = parse.urlunsplit( (scheme, new_netloc, path, query, frag)) auth_url = self.auth_url if self.version == "v2.0": # FIXME(chris): This should be better. while auth_url: if not self.auth_system or self.auth_system == 'keystone': auth_url = self._v2_auth(auth_url) else: auth_url = self._plugin_auth(auth_url) # Are we acting on behalf of another user via an # existing token? If so, our actual endpoints may # be different than that of the admin token. if self.proxy_token: if self.bypass_url: self.set_management_url(self.bypass_url) else: self._fetch_endpoints_from_auth(admin_url) # Since keystone no longer returns the user token # with the endpoints any more, we need to replace # our service account token with the user token. self.auth_token = self.proxy_token else: try: while auth_url: auth_url = self._v1_auth(auth_url) # In some configurations nova makes redirection to # v2.0 keystone endpoint. Also, new location does not contain # real endpoint, only hostname and port. except exceptions.AuthorizationFailure: if auth_url.find('v2.0') < 0: auth_url = auth_url + '/v2.0' self._v2_auth(auth_url) if self.bypass_url: self.set_management_url(self.bypass_url) elif not self.management_url: raise exceptions.Unauthorized('Nova Client') self._save_keys()
def update_actions(self, old_alarm=None): trustor_user_id = pecan.request.headers.get('X-User-Id') trustor_project_id = pecan.request.headers.get('X-Project-Id') roles = pecan.request.headers.get('X-Roles', '') if roles: roles = roles.split(',') else: roles = [] auth_plugin = pecan.request.environ.get('keystone.token_auth') for actions in (self.ok_actions, self.alarm_actions, self.insufficient_data_actions): if actions is not None: for index, action in enumerate(actions[:]): url = netutils.urlsplit(action) if self._is_trust_url(url): if '@' not in url.netloc: # We have a trust action without a trust ID, # create it trust_id = keystone_client.create_trust_id( trustor_user_id, trustor_project_id, roles, auth_plugin) netloc = '%s:delete@%s' % (trust_id, url.netloc) url = list(url) url[1] = netloc actions[index] = urlparse.urlunsplit(url) if old_alarm: for key in ('ok_actions', 'alarm_actions', 'insufficient_data_actions'): for action in getattr(old_alarm, key): url = netutils.urlsplit(action) if (self._is_trust_url(url) and url.password and action not in getattr(self, key)): keystone_client.delete_trust_id( url.username, auth_plugin)
def update_actions(self, old_alarm=None): trustor_user_id = pecan.request.headers.get("X-User-Id") trustor_project_id = pecan.request.headers.get("X-Project-Id") roles = pecan.request.headers.get("X-Roles", "") if roles: roles = roles.split(",") else: roles = [] auth_plugin = pecan.request.environ.get("keystone.token_auth") for actions in (self.ok_actions, self.alarm_actions, self.insufficient_data_actions): if actions is not None: for index, action in enumerate(actions[:]): url = netutils.urlsplit(action) if self._is_trust_url(url): if "@" not in url.netloc: # We have a trust action without a trust ID, # create it trust_id = keystone_client.create_trust_id( pecan.request.cfg, trustor_user_id, trustor_project_id, roles, auth_plugin ) netloc = "%s:delete@%s" % (trust_id, url.netloc) url = list(url) url[1] = netloc actions[index] = urlparse.urlunsplit(url) if old_alarm: new_actions = list( itertools.chain(self.ok_actions or [], self.alarm_actions or [], self.insufficient_data_actions or []) ) for action in itertools.chain( old_alarm.ok_actions or [], old_alarm.alarm_actions or [], old_alarm.insufficient_data_actions or [] ): if action not in new_actions: self.delete_trust(action)
def __init__(self, base, relative=None): self._has_token = False self._url = None self._url_parts = None self._loaded = False self._xml = None self._url_parts = None self._headers = None self._config = None if isinstance(base, six.string_types): base_url = base self._url_parts = list(parse.urlsplit(base_url)) elif isinstance(base, RequestBase): base_url = base.url self._has_token = base.has_token self._url_parts = base._url_parts[:] self._headers = base._headers self._config = base.config if relative: scheme, netloc, path, qs, fragment = parse.urlsplit(relative) if path: self._url_parts[2] = _join_plex(self._url_parts[2], path) if qs: data = parse.parse_qsl(self._url_parts[3]) + parse.parse_qsl(qs) self._url_parts[3] = parse.urlencode(data) else: # Strip of all non-token parts data = parse.parse_qsl(self._url_parts[3]) self._url_parts[3] = parse.urlencode([(x, y) for x, y in data if x == 'X-Plex-Token']) if not self._has_token: self._has_token = 'X-Plex-Token' in parse.parse_qs(self._url_parts[3]) self._url = parse.urlunsplit(self._url_parts)
def _strip_basic_auth(url): """Returns *url* with basic auth credentials removed. Also returns the basic auth username and password if they're present in *url*. E.g.: https://user:[email protected] => https://example.com *url* need not include basic auth credentials. :param url: url which may or may not contain basic auth credentials :type url: ``str`` :return: 3-``tuple`` of: * (``str``) -- *url* with any basic auth creds removed * (``str`` or ``NoneType``) -- basic auth username or ``None`` if basic auth username not given * (``str`` or ``NoneType``) -- basic auth password or ``None`` if basic auth password not given :rtype: ``tuple`` """ url_parts = parse.urlsplit(url) username = url_parts.username password = url_parts.password frags = list(url_parts) # swap out "user[:pass]@hostname" for "hostname" frags[1] = url_parts.hostname url = parse.urlunsplit(frags) return (url, username, password)
def _get_heat_signal_url(self, project_id=None): """Return a heat-api signal URL for this resource. This URL is not pre-signed, valid user credentials are required. If a project_id is provided, it is used in place of the original project_id. This is useful to generate a signal URL that uses the heat stack user project instead of the user's. """ stored = self.data().get('heat_signal_url') if stored is not None: return stored if self.id is None: # it is too early return url = self.client_plugin('heat').get_heat_url() host_url = urlparse.urlparse(url) path = self.identifier().url_path() if project_id is not None: path = project_id + path[path.find('/'):] url = urlparse.urlunsplit( (host_url.scheme, host_url.netloc, 'v1/%s/signal' % path, '', '')) self.data_set('heat_signal_url', url) return url
def _update_link_prefix(self, orig_url, prefix): if not prefix: return orig_url url_parts = list(parse.urlsplit(orig_url)) prefix_parts = list(parse.urlsplit(prefix)) url_parts[0:2] = prefix_parts[0:2] return parse.urlunsplit(url_parts)
def remove_version_from_href(href): """Removes the first api version from the href. Given: 'http://manila.example.com/v1.1/123' Returns: 'http://manila.example.com/123' Given: 'http://www.manila.com/v1.1' Returns: 'http://www.manila.com' Given: 'http://manila.example.com/share/v1.1/123' Returns: 'http://manila.example.com/share/123' """ parsed_url = parse.urlsplit(href) url_parts = parsed_url.path.split('/') # NOTE: this should match vX.X or vX expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)') for x in range(len(url_parts)): if expression.match(url_parts[x]): del url_parts[x] break new_path = '/'.join(url_parts) if new_path == parsed_url.path: msg = 'href %s does not contain version' % href LOG.debug(msg) raise ValueError(msg) parsed_url = list(parsed_url) parsed_url[2] = new_path return parse.urlunsplit(parsed_url)
def remove_version_from_href(href): """Removes the first api version from the href. Given: 'http://www.manila.com/v1.1/123' Returns: 'http://www.manila.com/123' Given: 'http://www.manila.com/v1.1' Returns: 'http://www.manila.com' """ parsed_url = parse.urlsplit(href) url_parts = parsed_url.path.split("/", 2) # NOTE: this should match vX.X or vX expression = re.compile(r"^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)") if expression.match(url_parts[1]): del url_parts[1] new_path = "/".join(url_parts) if new_path == parsed_url.path: msg = "href %s does not contain version" % href LOG.debug(msg) raise ValueError(msg) parsed_url = list(parsed_url) parsed_url[2] = new_path return parse.urlunsplit(parsed_url)
def _send_new_pending_email(instance): """ Send an email to settings.API_ACCESS_MANAGER_EMAIL with the contents of this API access request. """ context = { 'approval_url': urlunsplit( ( 'https' if settings.HTTPS == 'on' else 'http', instance.site.domain, reverse('admin:api_admin_apiaccessrequest_change', args=(instance.id,)), '', '', ) ), 'api_request': instance } message = render_to_string('api_admin/api_access_request_email_new_request.txt', context) try: send_mail( _(u'API access request from {company}').format(company=instance.company_name), message, settings.API_ACCESS_FROM_EMAIL, [settings.API_ACCESS_MANAGER_EMAIL], fail_silently=False ) except SMTPException: log.exception(u'Error sending API user notification email for request [%s].', instance.id)
def add_or_replace_parameter(url, name, new_value): """Add or remove a parameter to a given url >>> import w3lib.url >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php', 'arg', 'v') 'http://www.example.com/index.php?arg=v' >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg4', 'v4') 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3&arg4=v4' >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg3', 'v3new') 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3new' >>> """ parsed = urlsplit(url) args = parse_qsl(parsed.query, keep_blank_values=True) new_args = [] found = False for name_, value_ in args: if name_ == name: new_args.append((name_, new_value)) found = True else: new_args.append((name_, value_)) if not found: new_args.append((name, new_value)) query = urlencode(new_args) return urlunsplit(parsed._replace(query=query))
def query_lookupd(self): """ Trigger a query of the configured ``nsq_lookupd_http_addresses``. """ endpoint = self.lookupd_http_addresses[self.lookupd_query_index] self.lookupd_query_index = (self.lookupd_query_index + 1) % len(self.lookupd_http_addresses) # urlsplit() is faulty if scheme not present if '://' not in endpoint: endpoint = 'http://' + endpoint scheme, netloc, path, query, fragment = urlparse.urlsplit(endpoint) if not path or path == "/": path = "/lookup" params = cgi.parse_qs(query) params['topic'] = self.topic query = urlparse.urlencode(_utf8_params(params), doseq=1) lookupd_url = urlparse.urlunsplit((scheme, netloc, path, query, fragment)) req = tornado.httpclient.HTTPRequest( lookupd_url, method='GET', connect_timeout=self.lookupd_connect_timeout, request_timeout=self.lookupd_request_timeout) callback = functools.partial(self._finish_query_lookupd, lookupd_url=lookupd_url) self.http_client.fetch(req, callback=callback)
def get_url(self, path="/", query=None, fragment=None): if not self.started: return None return urlunsplit(("http" if not self.use_ssl else "https", "%s:%s" % (self.host, self.port), path, query, fragment))
def fix_links_in_lxml_tree(link): """Fix links in an lxml tree. This function is called by the rewrite_links method of an lxml tree, and is used to normalize links in a few ways. It makes links absolute, works around buggy URLs and nukes anchors. Example: html_tree.rewrite_links(fix_links_in_lxml_tree, base_href=my_url) Some URLS, like the following, make no sense: - https://www.appeals2.az.gov/../Decisions/CR20130096OPN.pdf. ^^^^ -- This makes no sense! The fix is to remove any extra '/..' patterns at the beginning of the path. Others have annoying anchors on the end, like: - http://example.com/path/#anchor Note that lxml has a method generally for this purpose called make_links_absolute, but we cannot use it because it does not work around invalid relative URLS, nor remove anchors. This is a limitation of Python's urljoin that will be fixed in Python 3.5 according to a bug we filed: http://bugs.python.org/issue22118 """ url_parts = urlsplit(link) url = urlunsplit( url_parts[:2] + (re.sub('^(/\.\.)+', '', url_parts.path),) + url_parts[3:] ) return url.split('#')[0]
def _parse_url(self, url, ssl=False): """Create a url from test data. If provided with a full URL, just return that. If SSL is requested set the scheme appropriately. Scheme and netloc are saved for later use in comparisons. """ parsed_url = urlparse.urlsplit(url) url_scheme = parsed_url[0] scheme = 'http' netloc = self.host if not url_scheme: if self.port: netloc = '%s:%s' % (self.host, self.port) if ssl: scheme = 'https' path = parsed_url[2] if self.prefix: path = '%s%s' % (self.prefix, path) full_url = urlparse.urlunsplit((scheme, netloc, path, parsed_url[3], '')) self.scheme = scheme self.netloc = netloc else: full_url = url self.scheme = url_scheme self.netloc = parsed_url[1] return full_url
def _parse_url(self, url): """Create a url from test data. If provided with a full URL, just return that. If SSL is requested set the scheme appropriately. Scheme and netloc are saved for later use in comparisons. """ query_params = self.test_data['query_parameters'] ssl = self.test_data['ssl'] parsed_url = urlparse.urlsplit(url) if not parsed_url.scheme: full_url = utils.create_url(url, self.host, port=self.port, prefix=self.prefix, ssl=ssl) # parse again to set updated netloc and scheme parsed_url = urlparse.urlsplit(full_url) self.scheme = parsed_url.scheme self.netloc = parsed_url.netloc if query_params: query_string = self._update_query_params(parsed_url.query, query_params) else: query_string = parsed_url.query return urlparse.urlunsplit((parsed_url.scheme, parsed_url.netloc, parsed_url.path, query_string, ''))
def _build_api_url(url, query): scheme, netloc, path, base_query, fragment = urlsplit(url) if base_query: query = '%s&%s' % (base_query, query) return urlunsplit((scheme, netloc, path, query, fragment))
def _send_decision_email(instance): """ Send an email to requesting user with the decision made about their request. """ context = { 'name': instance.user.username, 'api_management_url': urlunsplit( ( 'https' if settings.HTTPS == 'on' else 'http', instance.site.domain, reverse('api_admin:api-status'), '', '', ) ), 'authentication_docs_url': settings.AUTH_DOCUMENTATION_URL, 'api_docs_url': settings.API_DOCUMENTATION_URL, 'support_email_address': settings.API_ACCESS_FROM_EMAIL, 'platform_name': configuration_helpers.get_value('PLATFORM_NAME', settings.PLATFORM_NAME) } message = render_to_string( 'api_admin/api_access_request_email_{status}.txt'.format(status=instance.status), context ) try: send_mail( _('API access request'), message, settings.API_ACCESS_FROM_EMAIL, [instance.user.email], fail_silently=False ) instance.contacted = True except SMTPException: log.exception(u'Error sending API user notification email for request [%s].', instance.id)
def _deprecated_amqp_url(): """Allow for deprecating amqp_url setting over time. This warns and attempts to translate an amqp_url to something oslo_messaging can use to load a driver. """ url = cfg.CONF.amqp_url if not url: return LOG.warning(_LW( 'Use of amqp_url is deprecated. Please instead use options defined in ' 'oslo_messaging_rabbit to declare your AMQP connection.')) url = urlparse.urlsplit(url) if url.scheme == 'amqp': scheme = 'rabbit' else: scheme = url.scheme port = str(url.port or 5672) netloc = url.netloc if netloc.endswith(':'): netloc = netloc[:-1] out = urlparse.urlunsplit(( scheme, '%s:%s' % (netloc, port), url.path, '', '' )) return out
def urljoin(orig_url, url): # pragma: no cover new_url = urljoin(orig_url, url) if '../' not in new_url: return new_url # only needed in py2 as py3 urljoin resolves '../' parts = urlsplit(new_url) scheme, netloc, path, query, frag = parts path_parts = path.split('/') i = 0 n = len(path_parts) - 1 while i < n: if path_parts[i] == '..': del path_parts[i] n -= 1 if i > 0: del path_parts[i - 1] n -= 1 i -= 1 else: i += 1 if path_parts == ['']: path = '/' else: path = '/'.join(path_parts) parts = (scheme, netloc, path, query, frag) new_url = urlunsplit(parts) return new_url
def update_url_query(*args, **kwargs): """ Return a new URL with the query parameters of the URL updated based on the keyword arguments of the function call. If the argument already exists in the URL, it will be overwritten with the new value; if not, it will be added. However, if the new value is None, then any existing query parameters with that key will be removed without being replaced. The URL must be passed as the first positional argument of the function; it cannot be passed as a keyword argument. """ if not args: raise TypeError("URL must be passed as the first positional argument") url = args[0] scheme, netloc, path, query, fragment = urlsplit(url) qlist = parse_qsl(query) for key, value in kwargs.items(): # remove all key/value pairs from qlist that match this key qlist = [pair for pair in qlist if not pair[0] == key] # add this key/value pair to the qlist (unless it's None) if value is not None: qlist.append((key, value)) # bring it on back query = urlencode(qlist) return urlunsplit((scheme, netloc, path, query, fragment))
def get_latest_summary_url(cls, election_url): election_url = cls._url_ensure_trailing_slash(election_url) current_ver = cls.get_current_ver(election_url) # If we don't have current_ver, we can't determine a summary URL. if current_ver is None: return None if 'Web02' in election_url: election_url_parts = parse.urlsplit(election_url) election_url_parts = election_url_parts._replace(path="/".join( election_url_parts.path.split('/')[:3]), fragment='') else: election_url_parts = parse.urlsplit(election_url) new_paths = [ election_url_parts.path + '/' + current_ver + "/json/en/summary.json", election_url_parts.path + current_ver + "/Web01/en/summary.html", election_url_parts.path + current_ver + "/en/summary.html", ] for new_path in new_paths: latest_summary_url_parts = election_url_parts._replace( path=new_path) latest_summary_url = parse.urlunsplit(latest_summary_url_parts) latest_summary_url_response = requests.get(latest_summary_url) try: latest_summary_url_response.raise_for_status() except requests.exceptions.HTTPError: continue return latest_summary_url # If none of the expected paths succeed, return None. return None
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit(( scheme, netloc, path + '.dods', quote(self.id) + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session) raise_for_status(r) dds, data = r.body.split(b'\nData:\n', 1) dds = dds.decode(r.content_encoding or 'ascii') if self.shape: # skip size packing if self.dtype.char in 'SU': data = data[4:] else: data = data[8:] # calculate array size shape = tuple( int(np.ceil((s.stop-s.start)/float(s.step))) for s in index) size = int(np.prod(shape)) if self.dtype == np.byte: return np.fromstring(data[:size], 'B') elif self.dtype.char in 'SU': out = [] for word in range(size): n = np.fromstring(data[:4], '>I') # read length data = data[4:] out.append(data[:n]) data = data[n + (-n % 4):] return np.array([ text_type(x.decode('ascii')) for x in out ], 'S') else: return np.fromstring(data, self.dtype).reshape(shape)
def _winrm_connect(self): ''' Establish a WinRM connection over HTTP/HTTPS. ''' port = self.port or 5986 vvv("ESTABLISH WINRM CONNECTION FOR USER: %s on PORT %s TO %s" % \ (self.user, port, self.host), host=self.host) netloc = '%s:%d' % (self.host, port) cache_key = '%s:%s@%s:%d' % (self.user, hashlib.md5(self.password).hexdigest(), self.host, port) if cache_key in _winrm_cache: vvvv('WINRM REUSE EXISTING CONNECTION: %s' % cache_key, host=self.host) return _winrm_cache[cache_key] transport_schemes = [('plaintext', 'https'), ('plaintext', 'http')] # FIXME: ssl/kerberos if port == 5985: transport_schemes = reversed(transport_schemes) exc = None for transport, scheme in transport_schemes: endpoint = urlparse.urlunsplit((scheme, netloc, '/wsman', '', '')) vvvv('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint), host=self.host) protocol = Protocol(endpoint, transport=transport, username=self.user, password=self.password) try: protocol.send_message('') _winrm_cache[cache_key] = protocol return protocol except WinRMTransportError, exc: err_msg = str(exc) if re.search(r'Operation\s+?timed\s+?out', err_msg, re.I): raise errors.AnsibleError("the connection attempt timed out") m = re.search(r'Code\s+?(\d{3})', err_msg) if m: code = int(m.groups()[0]) if code == 401: raise errors.AnsibleError("the username/password specified for this server was incorrect") elif code == 411: _winrm_cache[cache_key] = protocol return protocol vvvv('WINRM CONNECTION ERROR: %s' % err_msg, host=self.host) continue
def keystoneclient(request, admin=False): user = request.user if admin: endpoint_type = 'adminURL' else: endpoint_type = getattr(settings, 'OPENSTACK_ENDPOINT_TYPE', 'internalURL') api_version = VERSIONS.get_active_version() # Take care of client connection caching/fetching a new client. # Admin vs. non-admin clients are cached separately for token matching. cache_attr = "_keystoneclient_admin" if admin \ else backend.KEYSTONE_CLIENT_ATTR if (hasattr(request, cache_attr) and (not user.token.id or getattr(request, cache_attr).auth_token == user.token.id)): conn = getattr(request, cache_attr) else: endpoint = _get_endpoint_url(request, endpoint_type) magic_tuple = netutils.urlsplit(endpoint) scheme, netloc, path, query, frag = magic_tuple port = magic_tuple.port if port is None: port = 80 new_netloc = netloc.replace(':%d' % port, ':%d' % (35357, )) endpoint = urlparse.urlunsplit((scheme, new_netloc, path, query, frag)) insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False) cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', None) LOG.debug("Creating a new keystoneclient connection to %s." % endpoint) remote_addr = request.environ.get('REMOTE_ADDR', '') conn = api_version['client'].Client(token=user.token.id, endpoint=endpoint, original_ip=remote_addr, insecure=insecure, cacert=cacert, auth_url=endpoint, debug=settings.DEBUG) setattr(request, cache_attr, conn) return conn
def __getitem__(self, index): # build download url index = combine_slices(self.slice, fix_slice(index, self.shape)) scheme, netloc, path, query, fragment = urlsplit(self.baseurl) url = urlunsplit((scheme, netloc, path + '.dods', quote(self.id) + hyperslab(index) + '&' + query, fragment)).rstrip('&') # download and unpack data logger.info("Fetching URL: %s" % url) r = GET(url, self.application, self.session, timeout=self.timeout, verify=self.verify) raise_for_status(r) dds, data = safe_dds_and_data(r, self.user_charset) # Parse received dataset: dataset = build_dataset(dds) dataset.data = unpack_data(BytesReader(data), dataset) return dataset[self.id].data
def call_for(self, endpoint, to, **values): """ Initiate a Twilio call. Parameters ---------- endpoint : `str` The view endpoint, as would be passed to :py:func:`flask.url_for`. to : `str` The destination phone number. values : `dict` Additional keyword arguments to pass to :py:func:`flask.url_for`. Returns ------- call : `twilio.rest.resources.Call` An object representing the call in progress. """ # Extract keyword arguments that are intended for `calls.create` # instead of `url_for`. values = dict(values, _external=True) from_ = values.pop('from_', None) or current_app.config['TWILIO_FROM'] # Construct URL for endpoint. url = url_for(endpoint, **values) # If we are not in debug or testing mode and a secret key is set, then # add HTTP basic auth information to the URL. The username is `twilio`. # The password is a random string that has been signed with # `itsdangerous`. if not(current_app.debug or current_app.testing or self.signer is None): urlparts = list(urlsplit(url)) token = ''.join(rand.choice(letters_and_digits) for i in range(32)) password = self.signer.sign(token).decode() urlparts[1] = 'twilio:' + password + '@' + urlparts[1] url = urlunsplit(urlparts) # Issue phone call. return self.client.calls.create(to=to, from_=from_, url=url)
def create_dest_url(self, msg, client, item): """Create the destination URL and the connection parameters.""" defaults = self.config[client] info_dict = dict() for key in ['host', 'directory', 'filepattern']: try: info_dict[key] = item[key] except KeyError: info_dict[key] = defaults[key] connection_parameters = item.get('connection_parameters', defaults.get('connection_parameters')) host = info_dict['host'] path = os.path.join(info_dict['directory'], info_dict['filepattern']) mda = msg.data.copy() for key, aliases in defaults.get('aliases', {}).items(): if key in mda: mda[key] = aliases.get(mda[key], mda[key]) path = compose(path, mda) parts = urlsplit(host) host_path = urlunsplit( (parts.scheme, parts.netloc, path, parts.query, parts.fragment)) return host_path, connection_parameters
def safe_url_string(url, encoding='utf8', path_encoding='utf8'): """Convert the given URL into a legal URL by escaping unsafe characters according to RFC-3986. If a bytes URL is given, it is first converted to `str` using the given encoding (which defaults to 'utf-8'). 'utf-8' encoding is used for URL path component (unless overriden by path_encoding), and given encoding is used for query string or form data. When passing an encoding, you should use the encoding of the original page (the page from which the URL was extracted from). Calling this function on an already "safe" URL will return the URL unmodified. Always returns a native `str` (bytes in Python2, unicode in Python3). """ # Python3's urlsplit() chokes on bytes input with non-ASCII chars, # so let's decode (to Unicode) using page encoding: # - it is assumed that a raw bytes input comes from a document # encoded with the supplied encoding (or UTF8 by default) # - if the supplied (or default) encoding chokes, # percent-encode offending bytes parts = urlsplit(to_unicode(url, encoding=encoding, errors='percentencode')) # quote() in Python2 return type follows input type; # quote() in Python3 always returns Unicode (native str) return urlunsplit(( to_native_str(parts.scheme), to_native_str(parts.netloc.encode('idna')), # default encoding for path component SHOULD be UTF-8 quote(to_bytes(parts.path, path_encoding), _safe_chars), # encoding of query and fragment follows page encoding # or form-charset (if known and passed) quote(to_bytes(parts.query, encoding), _safe_chars), quote(to_bytes(parts.fragment, encoding), _safe_chars), ))
def http_request(self, path="/", method="GET", host=None, port=None, json=False, data=None): """ perform a HTTP request :param path: str, path within the reqest, e.g. "/api/version" :param method: str, HTTP method :param host: str, if None, set self.get_IPv4s()[0] :param port: str or int, if None, set to self.get_ports()[0] :param json: bool, should we expect json? :param data: data to send (can be dict, list, str) :return: dict """ host = host or self.get_IPv4s()[0] port = port or self.get_ports()[0] url = urlunsplit(("http", "%s:%s" % (host, port), path, "", "")) return self.http_session.request(method, url, json=json, data=data)
def api_url(host, port, endpoint): """Returns a joined url. Takes host, port and endpoint and generates a valid emby API url. :param host: Hostname of the emby server :param port: Portnumber of the emby server :param endpoint: API endpoint :type host: str :type port: int :type endpoint: str :returns: Full API url :rtype: str """ joined = urljoin('{0}:{1}'.format(host, port), endpoint) scheme, netloc, path, query_string, fragment = urlsplit(joined) query_params = parse_qs(query_string) query_params['format'] = ['json'] new_query_string = urlencode(query_params, doseq=True) return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def mask_password_from_url( url ): """ Masks out passwords from connection urls like the database connection in galaxy.ini >>> mask_password_from_url( 'sqlite+postgresql://user:password@localhost/' ) 'sqlite+postgresql://user:********@localhost/' >>> mask_password_from_url( 'amqp://*****:*****@localhost' ) 'amqp://*****:*****@localhost' >>> mask_password_from_url( 'amqp://localhost') 'amqp://localhost' """ split = urlparse.urlsplit(url) if split.password: if url.count(split.password) == 1: url = url.replace(split.password, "********") else: # This can manipulate the input other than just masking password, # so the previous string replace method is preferred when the # password doesn't appear twice in the url split = split._replace(netloc=split.netloc.replace("%s:%s" % (split.username, split.password), '%s:********' % split.username)) url = urlparse.urlunsplit(split) return url
def _cs_request(self, url, method, **kwargs): if not self.management_url: self.authenticate() if url is None: # To get API version information, it is necessary to GET # a nova endpoint directly without "v2/<tenant-id>". magic_tuple = parse.urlsplit(self.management_url) scheme, netloc, path, query, frag = magic_tuple path = re.sub(r'v[1-9](\.[1-9][0-9]*)?/[a-z0-9]+$', '', path) url = parse.urlunsplit((scheme, netloc, path, None, None)) else: if self.service_catalog and not self.bypass_url: url = self.get_service_url(self.service_type) + url else: url = self.management_url + url # Perform the request once. If we get a 401 back then it # might be because the auth token expired, so try to # re-authenticate and try again. If it still fails, bail. try: kwargs.setdefault('headers', {})['X-Auth-Token'] = self.auth_token if self.projectid: kwargs['headers']['X-Auth-Project-Id'] = self.projectid resp, body = self._time_request(url, method, **kwargs) return resp, body except exceptions.Unauthorized as e: try: # first discard auth token, to avoid the possibly expired # token being re-used in the re-authentication attempt self.unauthenticate() # overwrite bad token self.keyring_saved = False self.authenticate() kwargs['headers']['X-Auth-Token'] = self.auth_token resp, body = self._time_request(url, method, **kwargs) return resp, body except exceptions.Unauthorized: raise e
def remove_version_from_href(href): """ Removes the first api version from the href. """ parsed_url = urlparse.urlsplit(href) url_parts = parsed_url.path.split('/', 2) # NOTE: this should match vX.X or vX expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)') if expression.match(url_parts[1]): del url_parts[1] new_path = '/'.join(url_parts) if new_path == parsed_url.path: msg = _('href %s does not contain version') % href LOG.debug(msg) raise ValueError(msg) parsed_url = list(parsed_url) parsed_url[2] = new_path return urlparse.urlunsplit(parsed_url)
def verify_hostname(self, req_args, *args, **kwargs): net_agent = self.state.get_agent() url = urlunsplit( ('http', req_args['value'], 'robots.txt', None, None)).encode() try: resp = yield net_agent.request(b'GET', url) body = yield readBody(resp) server_h = resp.headers.getRawHeaders(b'Server', [None])[-1].lower() if not body.startswith( b'User-agent: *') or server_h != b'globaleaks': raise EnvironmentError('Response unexpected') except Exception as e: # Convert networking failures into a generic response if is_common_net_error(self.state.tenant_cache[self.request.tid], e): raise errors.ExternalResourceError() raise e
def clean_url(url): """ Returns an cleaned url starting with a scheme and folder with trailing / or an empty string """ if url and url.strip(): url = xhtml_unescape(url.strip()) if '://' not in url: url = '//' + url scheme, netloc, path, query, fragment = parse.urlsplit(url, 'http') if not path: path += '/' cleaned_url = parse.urlunsplit((scheme, netloc, path, query, fragment)) else: cleaned_url = '' return cleaned_url
def _get_heat_signal_url(self): """Return a heat-api signal URL for this resource. This URL is not pre-signed, valid user credentials are required. """ stored = self.data().get('heat_signal_url') if stored is not None: return stored if self.id is None: # it is too early return url = self.client_plugin('heat').get_heat_url() host_url = urlparse.urlparse(url) path = self.identifier().url_path() url = urlparse.urlunsplit( (host_url.scheme, host_url.netloc, 'v1/%s/signal' % path, '', '')) self.data_set('heat_signal_url', url) return url
def update_query_parameters(url, query_parameters): """ Return url with updated query parameters. Arguments: url (str): Original url whose query parameters need to be updated. query_parameters (dict): A dictionary containing query parameters to be added to course selection url. Returns: (slug): slug identifier for the identity provider that can be used for identity verification of users associated the enterprise customer of the given user. """ scheme, netloc, path, query_string, fragment = urlsplit(url) url_params = parse_qs(query_string) # Update url query parameters url_params.update(query_parameters) return urlunsplit( (scheme, netloc, path, urlencode(sorted(url_params.items()), doseq=True), fragment), )
def is_allowed(self, request, timeout=None): s, n, p, q, f = urlsplit(request.url) if n in self.domain_blacklist: self.logger.error("Blocking request to a blacklisted domain: %r" % n) return False #: if set to not follow the robots.txt if not self.obey_robots_txt: return True robots_url = urlunsplit((s, n, 'robots.txt', None, None)) try: access_rules = self.robots_registry[robots_url] except KeyError: access_rules = self.load_rules_from_url(robots_url, timeout) if access_rules is None: # error - everybody welcome return True user_agent = request.headers.get('User-Agent', '*') allowed = access_rules.can_fetch(user_agent, request.url) if not allowed: return False request_rate = access_rules.request_rate(user_agent) if request_rate is None: #: No worries :) return True current_time = time.time() diff_time = current_time - access_rules.mtime() delay = request_rate.seconds / request_rate.requests if isinstance(delay, (int, float)) and not diff_time >= delay: self.logger.debug("Waiting on request for [%r] seconds!" % delay) time.sleep(delay) #: Update the access time value access_rules.modified()
def hashed_name(self, name, content=None, filename=None): parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() opened = False if content is None: absolute_path = finders.find(clean_name) print( f"Content is none; absolute_path: {absolute_path} / filename: {filename} / name: {name}" ) try: content = open(absolute_path, "rb") except (IOError, OSError) as e: if e.errno == errno.ENOENT: print(e) raise ValueError( "The file '%s' could not be found with %r." % (clean_name, self)) else: raise content = File(content) opened = True try: file_hash = self.file_hash(clean_name, content) finally: if opened: content.close() path, filename = os.path.split(clean_name) root, ext = os.path.splitext(filename) if file_hash is not None: file_hash = ".%s" % file_hash hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext)) unparsed_name = list(parsed_name) unparsed_name[2] = hashed_name # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax if "?#" in name and not unparsed_name[3]: unparsed_name[2] += "?" return urlunsplit(unparsed_name)
def proxy_request(self, req): headers = self.get_headers(req) url = urlparse.urlunsplit(( self.get_scheme(req), self.get_host(req), self.get_path_info(req), self.get_query_string(req), '')) h = self.create_http_client(req) resp, content = h.request( url, method=req.method, headers=headers, body=req.body ) if resp.status == 200: LOG.debug(str(resp)) return self.create_response(req, resp, content) elif resp.status == 403: LOG.warning( 'The remote metadata server responded with Forbidden. This ' 'response usually occurs when shared secrets do not match.') return webob.exc.HTTPForbidden() elif resp.status == 400: return webob.exc.HTTPBadRequest() elif resp.status == 404: return webob.exc.HTTPNotFound() elif resp.status == 409: return webob.exc.HTTPConflict() elif resp.status == 500: msg = ( 'Remote metadata server experienced an internal server error.' ) LOG.warning(msg) explanation = six.text_type(msg) return webob.exc.HTTPInternalServerError(explanation=explanation) else: raise Exception(_('Unexpected response code: %s') % resp.status)
def create_url(base_url, host, port=None, prefix='', ssl=False): """Given pieces of a path-based url, return a fully qualified url.""" scheme = 'http' # A host with : in it at this stage is assumed to be an IPv6 # address of some kind (they come in many forms). Port should # already have been stripped off. if ':' in host and not (host.startswith('[') and host.endswith(']')): host = '[%s]' % host if port and not _port_follows_standard(port, ssl): netloc = '%s:%s' % (host, port) else: netloc = host if ssl: scheme = 'https' parsed_url = urlparse.urlsplit(base_url) query_string = parsed_url.query path = parsed_url.path # Guard against a prefix of None or the url already having the # prefix. Without the startswith check, the tests in prefix.yaml # fail. This is a pragmatic fix which does this for any URL in a # test request that does not have a scheme and does not # distinguish between URLs in a gabbi test file and those # generated by the server. Idealy we would not mutate nor need # to check URLs returned from the server. Doing that, however, # would require more complex data handling than we have now and # this covers most common cases and will be okay until someone # reports a bug. if prefix and not path.startswith(prefix): prefix = prefix.rstrip('/') path = path.lstrip('/') path = '%s/%s' % (prefix, path) return urlparse.urlunsplit((scheme, netloc, path, query_string, ''))
def base_url(self): if self._base_url is None: if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self._base_url = self.swift_conn.url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self._base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self._base_url = urlparse.urlunsplit(split_result) self._base_url = urlparse.urljoin(self._base_url, self.container_name) self._base_url += '/' else: self._base_url = self.override_base_url return self._base_url
def generate_return_url(self, return_to, uid, path=""): """ :param return_to: If it starts with '/' it's an absolute path otherwise a relative path. :param uid: :param path: The verify path """ if return_to.startswith("http"): up = urlsplit(return_to) _path = up.path else: up = None _path = return_to if not _path.startswith("/"): p = path.split("/") p[-1] = _path _path = "/".join(p) if up: _path = urlunsplit([up[0], up[1], _path, up[3], up[4]]) return create_return_url(_path, uid, **{self.query_param: "true"})
def _get_safe_url(url): """Gets version of *url* with basic auth passwords obscured. This function returns results suitable for printing and logging. E.g.: https://user:[email protected] => https://[email protected] :param url: a url :type url: ``str`` :return: *url* with password removed :rtype: ``str`` """ parts = urlsplit(url) if parts.username is None: return url else: frags = list(parts) if parts.port: frags[1] = '{0}@{1}:{2}'.format(parts.username, parts.hostname, parts.port) else: frags[1] = '{0}@{1}'.format(parts.username, parts.hostname) return urlunsplit(frags)
def _insertUsernameIntoURL(self, url, username): """ If a particular account has to be used to log into the server specified in 'url', you can inject a "<username>@" using this function, e.g.: url = 'svn+ssh://svnext/Libraries/Spam/42.0' url = insertUsernameIntoURL( url, 'monthy' ) will return 'svn+ssh://monthy@svnext/Libraries/Spam/42.0'. Note: The function works with other protocols such as "http://" as well, e.g. as used by Git. """ Any.requireIsTextNonEmpty(url) urlData = list(parse.urlsplit(url)[:]) urlData[1] = '%s@%s' % (username, urlData[1]) result = parse.urlunsplit(urlData) Any.requireIsTextNonEmpty(result) return result
def percent_encode_host(url): """ Convert the host of uri formatted with to_uri() to have a %-encoded host instead of punycode host The rest of url should be unchanged """ # only continue if punycode encoded if 'xn--' not in url: return url parts = urlsplit(url) domain = parts.netloc.encode('utf-8') try: domain = domain.decode('idna') if six.PY2: domain = domain.encode('utf-8', 'ignore') except: # likely already encoded, so use as is pass domain = quote(domain) #, safe=r':\/') return urlunsplit((parts[0], domain, parts[2], parts[3], parts[4]))
def remove_trailing_version_from_href(href): """Removes the api version from the href. Given: 'http://www.nova.com/compute/v1.1' Returns: 'http://www.nova.com/compute' Given: 'http://www.nova.com/v1.1' Returns: 'http://www.nova.com' """ parsed_url = urlparse.urlsplit(href) url_parts = parsed_url.path.rsplit('/', 1) # NOTE: this should match vX.X or vX expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)') if not expression.match(url_parts.pop()): LOG.debug('href %s does not contain version', href) raise ValueError(_('href %s does not contain version') % href) new_path = url_join(*url_parts) parsed_url = list(parsed_url) parsed_url[2] = new_path return urlparse.urlunsplit(parsed_url)
def open_dods(url, metadata=False, application=None, session=None, timeout=DEFAULT_TIMEOUT, verify=True): """Open a `.dods` response directly, returning a dataset.""" r = GET(url, application, session, timeout=timeout) raise_for_status(r) dds, data = r.body.split(b'\nData:\n', 1) dds = dds.decode(r.content_encoding or 'ascii') dataset = build_dataset(dds) stream = StreamReader(BytesIO(data)) dataset.data = unpack_data(stream, dataset) if metadata: scheme, netloc, path, query, fragment = urlsplit(url) dasurl = urlunsplit( (scheme, netloc, path[:-4] + 'das', query, fragment)) r = GET(dasurl, application, session, timeout=timeout, verify=verify) raise_for_status(r) das = r.text add_attributes(dataset, parse_das(das)) return dataset
def get_current_ver(cls, election_url): election_url_parts = parse.urlsplit( cls._url_ensure_trailing_slash(election_url)) if 'Web02' in election_url: cls.parsed_url = election_url_parts._replace( path="/".join(election_url_parts.path.split('/')[:3]) + "/current_ver.txt", fragment='') election_url_parts = cls.parsed_url else: election_url_parts = election_url_parts._replace( path=election_url_parts.path + "current_ver.txt") current_ver_url = parse.urlunsplit(election_url_parts) current_ver_response = requests.get(current_ver_url) try: current_ver_response.raise_for_status() except requests.exceptions.HTTPError: return None return current_ver_response.text
def split_term_source_accession(val): if val is None: return '', '' if not r_url.match(val): # no URL if ':' in val: val_l = val.split(':') vocab = ontology_map.get(val_l[0], val_l[0]) return vocab, '{}:{}'.format(vocab, val[len(val_l[0]) + 1:]) else: # no idea lgr.warn("Could not identify term source REF in: '%s'", val) return '', val else: try: # this is a URL, assume simple: last path segment is accession id url_s = urlsplit(val) urlpath, accession = posixsplit(url_s.path) term_source = urlunsplit((url_s[0], url_s[1], urlpath, url_s[3], url_s[4])) return ontology_map.get(term_source, term_source), accession except Exception as e: lgr.warn("Could not identify term source REF in: '%s' [%s]", val, exc_str(e)) return '', val