def get_redirect_url(self): parsed = urlparse(self.service) query = parse_qs(parsed.query) query['ticket'] = [self.ticket] query = [((k, v) if len(v) > 1 else (k, v[0])) for k, v in query.items()] parsed = ParseResult(parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlencode(query), parsed.fragment) return parsed.geturl()
def __init__(self, username=None, password=None, site=None, email=None): self.username = username self.password = password parse = list(urlparse(site if urlparse(site).scheme else '%s%s' % ('//', site))) parse[0] = b'https' for index, value in enumerate(parse): parse[index] = value.decode('utf-8') if isinstance(value, bytes) \ else value parse = ParseResult(*parse) self.url = parse.geturl() self.email = email
def parse(self, url, respar): res = urlparse(url) nl = res.path #print(res) if respar.path.rfind("/")>=0: nl = respar.path[0:respar.path.rfind("/")+1] + res.path res2 = ParseResult(scheme = res.scheme or respar.scheme, netloc = res.netloc or respar.netloc, path = nl, params='', query='', fragment='') url = res2.geturl() #print("parse", url, res2, respar) return (url, res2)
def download_key_http(address, port): url = ParseResult( scheme='http', # This seems to work well enough with both IPv6 and IPv4 netloc="[[%s]]:%d" % (address, port), path='/', params='', query='', fragment='') log.debug("Starting HTTP request") data = requests.get(url.geturl(), timeout=5).content log.debug("finished downloading %d bytes", len(data)) return data
def __init__(self, url): """Initialize the url object. Create a new Url object from either a well formed url string, a dict of key/values, or a ParseResult. Args: url (mixed): The value to generate the url from. """ if isinstance(url, ParseResult): self._parts = url elif isinstance(url, dict): if 'hostname' in url and 'netloc' not in url: netloc = url.pop('hostname') if 'port' in url: netloc += ':' + str(url.pop('port')) url['netloc'] = netloc if 'scheme' not in url: url['scheme'] = 'http' if 'username' in url: url['netloc'] = '{0}:{1}@{2}'.format( url.pop('username'), url.pop('password', ''), url['netloc']) if 'params' not in url: url['params'] = None if 'fragment' not in url: url['fragment'] = None if 'path' not in url: url['path'] = '/' if 'query' not in url: url['query'] = None self._parts = ParseResult(**url) elif isinstance(url, str): self._parts = urlparse(url)
def url(self) -> str: """Returns the full url requested.""" return urlunparse( ParseResult( self.scheme, self.host, self.path, '', self.query_string, self.fragment, ), )
def make_test_headers_and_path( app: 'Quart', path: str, headers: Optional[Union[dict, CIMultiDict]]=None, query_string: Optional[dict]=None, ) -> Tuple[CIMultiDict, str]: """Make the headers and path with defaults for testing. Arguments: app: The application to test against. path: The path to request. headers: Initial headers to send. query_string: To send as a dictionary. """ if headers is None: headers = CIMultiDict() elif isinstance(headers, CIMultiDict): headers = headers elif headers is not None: headers = CIMultiDict(headers) headers.setdefault('Remote-Addr', '127.0.0.1') headers.setdefault('User-Agent', 'Quart') headers.setdefault('host', app.config['SERVER_NAME'] or 'localhost') if query_string is not None: path = urlunparse( ParseResult( scheme='', netloc='', params='', path=path, query=urlencode(query_string), fragment='', ), ) return headers, path # type: ignore
def __Select_Driver(self, drivers_link: ParseResult) -> ResultSet: """ __Select_Driver(drivers_link: ParseResult) -> ResultSet 具体版本号驱动列表 :param drivers_link: 具体版本号驱动链接 :return: ResultSet """ html = requests.get(url=drivers_link.geturl()) for retry in range(3): if html.status_code == 200: break html = requests.get(url=drivers_link.geturl()) html.encoding = html.apparent_encoding soup = BeautifulSoup(html.text, 'html.parser') return soup.pre.select(selector='a')
def paginate_undl_results(self, items_per_page=25, limit=100): ''' set a search term for digitallibrary and use the api (such as it is) to get total results back and to paginate through those results if @limit is set, stop after @limit ''' count = 1 if count == 1: yield self.url while count + items_per_page < limit: o = urlparse(self.url) query_dict = dict(parse_qsl(o.query)) if 'jrec' in query_dict.keys(): count = int(query_dict['jrec']) + items_per_page + 1 else: count = items_per_page + 1 query_dict.update({'jrec': count}) query_dict = urlencode(query_dict, doseq=True) new_url = ParseResult(o.scheme, o.netloc, o.path, o.params, query_dict, o.fragment).geturl() yield new_url count += items_per_page self.url = new_url logger.debug(self.url)
def clean_url(url: str) -> str: url = unquote(url) # Extracting url info parsed_url = urlparse(url) # Extracting URL arguments from parsed URL get_args = parsed_url.query # Converting URL arguments to dict parsed_get_args = dict(parse_qsl(get_args)) # Merging URL arguments dict with new params parsed_get_args.pop('page', '') parsed_get_args.update({ k: dumps(v) for k, v in parsed_get_args.items() if isinstance(v, (bool, dict)) }) # Converting URL argument to proper query string encoded_get_args = urlencode(parsed_get_args, doseq=True) # Creating new parsed result object based on provided with new # URL arguments. Same thing happens inside of urlparse. new_url = ParseResult(parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, encoded_get_args, parsed_url.fragment).geturl() return new_url
def get_callback_url(self): from urllib.parse import urlparse, ParseResult import urllib.request, urllib.parse, urllib.error callback_url_parse = urlparse(self.callback) query_args = { 'fwd_fee': self.fwd_miners_fee or 0, 'input_fee': self.input_miners_fee or 0, 'value': self.value, 'input_address': self.input_address, 'confirmations': self.confirmations, 'transaction_hash': self.transaction_hash, 'input_transaction_hash': self.input_transaction_hash, 'destination_address': self.destination_address, 'payee_addresses': self.payee_addresses } if not callback_url_parse.query: url_query = urllib.parse.urlencode(query_args) else: url_query = callback_url_parse.query + '&' + urllib.parse.urlencode( query_args) callback_url = ParseResult( scheme=callback_url_parse.scheme, netloc=callback_url_parse.netloc, path=callback_url_parse.path, params=callback_url_parse.params, query=url_query, fragment=callback_url_parse.fragment).geturl() return callback_url
def _fields_to_pr(cls, fields): """Recompose back fields dict to ParseResult""" netloc = fields['username'] or '' if fields['password']: netloc += ':' + fields['password'] if netloc: netloc += '@' netloc += fields['hostname'] if fields['port']: if fields['hostname'].count(':') >= 2: # ipv6 -- need to enclose in [] netloc = '[%s]:%s' % (netloc, fields['port']) else: netloc += ':%s' % fields['port'] pr_fields = { f: fields[f] for f in cls._FIELDS if f not in ('hostname', 'password', 'username', 'port') } pr_fields['netloc'] = netloc pr_fields['params'] = '' # We need to quote the path pr_fields['path'] = urlquote(pr_fields['path']) # TODO: figure out what to do with query/fragment... one step at a time return ParseResult(**pr_fields)
def device_auth(cmd): ts = openapi.timestamp() query_args = device_login_query_str(ts, cmd) query = urllib.parse.urlencode(query_args) parse_result = ParseResult(scheme=URL_SCHEME, netloc=URL_NETLOC, path=URL_PATH, query=query, params=URL_PARAMETER, fragment=URL_FRAGMENT) url = urlunparse(parse_result) return url
def make_absolute_url(relative_url, absolute_base): """ Create an absolute URL for selenium testing given a relative URL. This will also replace the host of absolute_base with the host IP of this instance. Args: relative_url (str): A relative URL absolute_base (str): An absolute URL which contains the http port and scheme we need Returns: str: An absolute URL pointing to the live server instance """ # Swap out the hostname, which was set to 0.0.0.0 to allow external connections # Change it to use ip of this container instead absolute_pieces = urlparse(absolute_base) relative_pieces = urlparse(relative_url) host = socket.gethostbyname(socket.gethostname()) return ParseResult( absolute_pieces.scheme, "{host}:{port}".format(host=host, port=absolute_pieces.port), relative_pieces.path, relative_pieces.params, relative_pieces.query, relative_pieces.fragment, ).geturl()
def get_chunk_tags(chunks: Dict, attrs: str): """ Get tags for :param chunks: :param attrs: :return: """ tags = [] for chunk in chunks: resource_type = chunk['resource_type'] original_url = chunk['url'] parse_result = urlparse(original_url) path = parse_result.path # If under STATIC_URL rewrite using static tag so that we respect static file storage # options, eg. ManifestStaticFileStorage if settings.STATIC_URL and path.startswith(settings.STATIC_URL): try: path = static(path[len(settings.STATIC_URL):]) except ValueError: # Allow url's that aren't managed by static files - eg. this will happen # for ManifestStaticFileStorage if file is not in the manifest pass url = ParseResult(**dict(parse_result._asdict(), path=path)).geturl() if resource_type == 'js': tags.append(f'<script type="text/javascript" src="{url}" {attrs}></script>') if resource_type == 'css': tags.append(f'<link type="text/css" href="{url}" rel="stylesheet" {attrs}/>') return tags
def apply_transformation(cls, parsed_url: ParseResult) -> ParseResult: """Apply the actual transformation process to the url.""" assert parsed_url.hostname is not None # mypy workaround new_domain = parsed_url.hostname.replace(".m.wikipedia.org", ".wikipedia.org") return parsed_url._replace(netloc=new_domain)
def fetch_resource_by_uuid(gatekeeper_endpoint: ParseResult, workspace_dir: str, kind: Kind, uuid: str) -> Dict[str, Any]: """Fetch a resource by its uuid. Return `None` if nothing is found. Raise a RuntimeError exception when a error is detected within the gatekeeper's API.""" url = urljoin(gatekeeper_endpoint.geturl(), os.path.join(_get_path_from_kind(kind), uuid)) _LOGGER.info('Fetching a resource by uuid at %s', url) auth = 'Bearer ' + _get_workspace_token(workspace_dir) res_resp = requests.get(url, headers={ 'content-type': 'application/json', 'Authorization': auth }) try: res_resp.raise_for_status() except requests.exceptions.HTTPError as exc_notfound: _LOGGER.exception( 'Failed to retrieve a resource at %s ' '(status code = %d)', res_resp.url, res_resp.status_code) if exc_notfound.response.status_code == 404: return None else: raise tmp = res_resp.json() if not isinstance(tmp, dict) or len(tmp) <= 0: exc = RuntimeError('The returned json is malformed: {}'.format(tmp)) _LOGGER.error('Error while fetching a resource using an uuid: %s', exc) raise exc _LOGGER.info('Succeed to retrieve the resource %s (status code = %d)', res_resp.url, res_resp.status_code) if kind.name in tmp: # the resource is boxed tmp = tmp[kind.name] return tmp
def get_safe_url(request, param_name, fallback_url=None): url = request.GET.get(param_name) or request.POST.get(param_name) allowed_hosts = settings.ALLOWED_HOSTS require_https = request.is_secure() if url: if settings.DEBUG: # In DEBUG mode the network location part `127.0.0.1:8000` contains # a port and fails the validation of `is_safe_url` since it's not a # member of `allowed_hosts`: # https://github.com/django/django/blob/23946bd/django/utils/http.py#L393 # As a quick fix, we build a new URL without the port. # pylint: disable=C0415 from urllib.parse import urlparse, ParseResult url_info = urlparse(url) url_without_port = ParseResult( scheme=url_info.scheme, netloc=url_info.hostname, path=url_info.path, params=url_info.params, query=url_info.query, fragment=url_info.fragment, ).geturl() if is_safe_url(url_without_port, allowed_hosts, require_https): return url else: if is_safe_url(url, allowed_hosts, require_https): return url return fallback_url
def validate_url(url, belongs_to=None): try: parsed_url = urlparse(url.rstrip('/')) except Exception: return False if parsed_url.scheme not in ('http', 'https', '') or \ (not belongs_to and not parsed_url.netloc) or \ (not parsed_url.netloc and not parsed_url.path): # skipping not http related resources (e.g. file:// etc) and empty urls return False if belongs_to: # make complete url out of relative/not full one # skip query string new_scheme = parsed_url.scheme if parsed_url.scheme else belongs_to.scheme new_netloc = parsed_url.netloc if parsed_url.netloc else belongs_to.netloc parsed_url = ParseResult(scheme=new_scheme, netloc=new_netloc, path=parsed_url.path, params='', query='', fragment='') if belongs_to.netloc.replace('www.', '') != parsed_url.netloc.replace('www.', ''): # given url does not belong to the root domain # skipping also subdomains as they can be really different resources # containing their own sitemaps return False return parsed_url
def add_url_params(url, params): """ Add GET params to provided URL being aware of existing. :param url: string of target URL :param params: dict containing requested params to be added :return: string with updated URL >> url = 'http://stackoverflow.com/test?answers=true' >> new_params = {'answers': False, 'data': ['some','values']} >> add_url_params(url, new_params) 'http://stackoverflow.com/test?data=some&data=values&answers=false' """ # Unquoting URL first so we don't loose existing args url = unquote(url) # Extracting url info parsed_url = urlparse(url) # Extracting URL arguments from parsed URL get_args = parsed_url.query # Converting URL arguments to dict parsed_get_args = dict(parse_qsl(get_args)) # Merging URL arguments dict with new params parsed_get_args.update(params) # Order the args ordered_args = get_ordered_args(parsed_get_args) # Converting URL argument to proper query string encoded_get_args = urlencode(ordered_args, doseq=True) # Creating new parsed result object based on provided with new # URL arguments. Same thing happens inside of urlparse. new_url = ParseResult( parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, encoded_get_args, parsed_url.fragment ).geturl() return new_url
def spoof_url(url: str, req_context: RequestContext) -> str: default_domain = req_context.target_parsed.netloc default_scheme = req_context.target_parsed.scheme def normalize_url(target_url: str): result = target_url result.replace('www.', '') #? Remove www. if result.split( '/' )[0] == default_domain: #? Check if first segment of 'path' is the default_domain result = f'{default_scheme}://' + result #? Add http(s):// in front if it is a url like (www.google.com/path) return result targetUrl = urlparse(normalize_url(url)) if targetUrl.scheme == '': targetUrl = targetUrl._replace(scheme=default_scheme) if targetUrl.netloc == '': targetUrl = targetUrl._replace(netloc=default_domain) spoofedTargetURL = urlunparse(targetUrl) spoofedURL = ParseResult( scheme= 'http', #? Always use http cause' I can't be bothered to run out and get a certificate netloc=req_context.own_host, path='/Request', query=qs_encode({'targetUrl': escape_url_for_qs(spoofedTargetURL)}), params=targetUrl.params, fragment=targetUrl.fragment) return urlunparse(spoofedURL)
def suggestions(): """GET /suggestions end-point. Accepted Query Parameters: q: the prefix. limit: the number of results by page. offset: the position of the first result in the list of all suggestions. """ prefix = request.args.get('q', '') suggestions = autocompleter.suggest(prefix) total = len(suggestions) try: limit = int(request.args.get('limit', total)) offset = int(request.args.get('offset', 0)) except ValueError: abort(400) response = { 'total': len(suggestions), 'limit': limit, 'offset': offset, 'results': suggestions[offset:offset + limit] } if offset < total - limit: next_query = urlencode({'limit': limit, 'offset': offset + limit}) next_url = urlunparse( ParseResult(scheme='', netloc='', path='suggestions', params='', query=next_query, fragment='')) response['next'] = next_url return jsonify(response)
def filter_query_params(url: str, parsed: up.ParseResult) -> up.ParseResult: """ Remove the following query params from an URL: * ``sid=\w+``: SIDs are mostly used by magento to track the users when cookies are disabled * ``s=\w{32}``: same as SID, but from vBulletin sites * ``replytocom=\d+``: used by wordpress when clicking on "answer" from a comment :param url: the url :return: the url without sid and the likes """ if any(l in parsed.query for l in ['s=', 'sid=', 'replytocom=']): # TODO: here, the behavior of parse is inconsistant/changes the URL # e.g.: # >>> up.parse_qsl('a=%7E_%7E%3B') # [('a', '~_~;')] # >>> up.urlencode([('a', '~_~;')]) # 'a=~_~%3B' qs = up.parse_qsl(parsed.query) new_qs = up.urlencode([ q for q in qs if not ((q[0] == 'sid') or # magento (q[0] == 's' and len(q[1]) == 32) or # vBulletin (q[0] == 'replytocom') # wordpress ) ]) return parsed._replace(query=new_qs) return parsed
def _normalize_url(self, url): argnames = ['scheme', 'netloc', 'path', 'params', 'query', 'fragment'] inheritable = ['scheme', 'netloc', 'path'] parsed_current_url = urlparse(self.current_url) parsed_url = urlparse(url) args = [] for argname in argnames: value = getattr(parsed_url, argname, None) if not value and argname in inheritable: value = getattr(parsed_current_url, argname, '') args.append(value) pr = ParseResult(*args) normalized = pr.geturl() return normalized
async def qrcode_api( request: Request, *, text: str, size: int = 200, logo: Optional[HostUrl] = None, encode: ReturnEncode = ReturnEncode.raw, level: QRCodeLevel = QRCodeLevel.M, bgcolor: Color = Color("FFFFFF"), fgcolor: Color = Color("000000"), fun: str = "qrcode", ): qr = await QRInfo.new( text, size=size, logo=logo, level=level, bgcolor=bgcolor, fgcolor=fgcolor ) qr.url = ParseResult( # type:ignore scheme=request.url.scheme, netloc=request.url.netloc, path=f"temp/{qr.path.relative_to(TempFile.path)}", params="", query="", fragment="", ).geturl() return ( qr if encode == ReturnEncode.json else Response(headers={"Location": qr.url}, status_code=302) if encode == ReturnEncode.raw else Response(content=f"{fun}({qr.json()})", media_type="text/javascript") )
def url(self) -> str: """Returns the full url requested.""" return urlunparse( ParseResult( self.scheme, self.host, self.path, '', self.query_string.decode('ascii'), '', ), )
def fetch_resource_by_uuid(gatekeeper_endpoint: ParseResult, workspace_dir: str, kind: Kind, uuid: str) -> Dict[str, Any]: """Fetch a resource by its uuid. Return `None` if nothing is found. Raise a RuntimeError exception when a error is detected within the gatekeeper's API.""" url = urljoin(gatekeeper_endpoint.geturl(), os.path.join(_get_path_from_kind(kind), uuid)) _LOGGER.info('Fetching a resource by uuid at %s', url) auth = 'Bearer ' + _get_workspace_token(workspace_dir) res_resp = requests.get(url, headers={'content-type': 'application/json', 'Authorization': auth}) try: res_resp.raise_for_status() except requests.exceptions.HTTPError as exc_notfound: _LOGGER.exception('Failed to retrieve a resource at %s ' '(status code = %d)', res_resp.url, res_resp.status_code) if exc_notfound.response.status_code == 404: return None else: raise tmp = res_resp.json() if not isinstance(tmp, dict) or len(tmp) <= 0: exc = RuntimeError('The returned json is malformed: {}'.format(tmp)) _LOGGER.error('Error while fetching a resource using an uuid: %s', exc) raise exc _LOGGER.info('Succeed to retrieve the resource %s (status code = %d)', res_resp.url, res_resp.status_code) if kind.name in tmp: # the resource is boxed tmp = tmp[kind.name] return tmp
def handle_authcode(request, client, redirection_uri, state=None): parts = urlparse(redirection_uri.uri) qparams = dict(parse_qsl(parts.query)) user_id = authenticated_userid(request) auth_code = Oauth2Code(client, user_id) db.add(auth_code) db.flush() qparams['code'] = auth_code.authcode if state: qparams['state'] = state parts = ParseResult( parts.scheme, parts.netloc, parts.path, parts.params, urlencode(qparams), '') return HTTPFound(location=parts.geturl())
def fetch_gitlab_versions(url: ParseResult) -> List[Version]: match = GITLAB_API.match(url.geturl()) if not match: return [] domain = match.group("domain") project_id = match.group("project_id") gitlab_url = f"https://{domain}/api/v4/projects/{project_id}/repository/tags" info(f"fetch {gitlab_url}") resp = urllib.request.urlopen(gitlab_url) json_tags = json.loads(resp.read()) if len(json_tags) == 0: raise VersionError("No git tags found") releases = [] tags = [] for tag in json_tags: name = tag["name"] assert isinstance(name, str) if tag["release"]: # TODO: has gitlab preleases? releases.append(Version(name)) else: tags.append(Version(name)) # if no release is found, use latest tag if releases == []: return tags return releases
def generate_provider_url(self, schema="dubbo", host="127.0.0.1", port="20880", sapplication=None, service=None, methods=None, validate=True, defaultFilter=None, dubbo_v="2.6.8"): if methods is None: methods = [] query_map = { "applicaton": sapplication, "bean.name": service, "default.service.filter": defaultFilter, "default.validation": validate, "dubbo": dubbo_v, "generic": False, "interface": service, "methods": ",".join(methods), "side": "provider", "timestamp": self.timestamp } query = urlencode(query_map) f = ParseResult(scheme=schema, netloc='%s:%s' % (host, port), path=service, params='', query=query, fragment='') return urlunparse(f)
def response_url_redirect(url, **kwargs): parsed_url = urlparse(url) query_list = parse_qsl(parsed_url.query) for key, value in kwargs.items(): query_list.append((key, value)) new_url = ParseResult(parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, urlencode(query_list), parsed_url.fragment) return skygear.Response(status=302, headers=[('Location', new_url.geturl())])
def apply_transformation(cls, parsed_url: ParseResult) -> ParseResult: """Apply the actual transformation process to the url.""" query_params = parse_qs(parsed_url.query, keep_blank_values=True) query_params.pop("fbclid", None) return parsed_url._replace(query=urlencode(query_params, doseq=True))
def _limit_and_offset(uri, limit=None, offset=None): """ Set limit and/or offset query parameters of the given URI. """ parts = urlparse(uri) query = parse_qs(parts.query) if limit is None: query.pop('limit', None) else: query['limit'] = limit if offset is None: query.pop('offset', None) else: query['offset'] = offset # in Python 2, urllib expects encoded byte-strings if six.PY2: new_query = {} for k, v in query.items(): if isinstance(v, list): v = [unicode(element).encode('utf-8') for element in v] elif isinstance(v, str): v = unicode(v).encode('utf-8') new_query[unicode(k).encode('utf-8')] = v query = new_query new_query_string = urlencode(query, doseq=True) return urlunparse( ParseResult(scheme=parts.scheme, netloc=parts.netloc, path=parts.path, params=parts.params, query=new_query_string, fragment=parts.fragment))
def get_redacted_url(result: ParseResult) -> ParseResult: if result.password: username: str = result.username hostname: str = result.hostname result = result._replace( netloc=f"{username}:{'*' * len(result.password)}@{hostname}") return result
def __init__(self, hass: HomeAssistant, entry: ConfigEntry) -> None: """Initialize the data object.""" super().__init__(hass, _LOGGER, name="SolarLog", update_interval=timedelta(seconds=60)) host_entry = entry.data[CONF_HOST] url = urlparse(host_entry, "http") netloc = url.netloc or url.path path = url.path if url.netloc else "" url = ParseResult("http", netloc, path, *url[3:]) self.unique_id = entry.entry_id self.name = entry.title self.host = url.geturl()
def join(self, path): """ assumes this object is the base URL or base path. If the path is relative, it should be appended to the base. If the path is absolute, it should be added to the connection details of self. If the path already contains connection details and the connection details differ from self, raise an error. """ pathAsString = str(path) if not path or not pathAsString: return self path = URL.objectify(path) if ((path.scheme and self.scheme and path.scheme != self.scheme) or (path.hostname and self.hostname and path.hostname != self.hostname) or (path.port and self.port and path.port != self.port)): raise ValueError("%s can't be joined with %s" % (self, path)) if path.path[0] == '/': ret_path = uc2utf8(path.path) else: sep = "/" if self.path.endswith("/"): sep = "" ret_path = "%s%s%s" % (self.path, sep, uc2utf8(path.path)) return URL( ParseResult(self.scheme or path.scheme, self.netloc or path.netloc, ret_path, path.params, path.query, path.fragment))
def dispatch(self, request, *args, **kwargs): if not test_func_email_required(request.user): messages.add_message( request, messages.INFO, "You need to have an email address on file before you can do that.", ) # Remember where they were trying to go, so we can redirect them # back after they agree. There's logic in TermsView to pick up on # this parameter. next_path = request.path next_param = urlencode({REDIRECT_FIELD_NAME: next_path}) path = reverse_lazy("users:email_change") new_url = ParseResult( scheme="", netloc="", path=str(path), params="", query=str(next_param), fragment="", ).geturl() return HttpResponseRedirect(new_url) return super(EmailRequired, self).dispatch(request, *args, **kwargs)
def get_direct_url(url, headers): """Gets the zip direct download link from the project download page""" direct_download_url = href_from_link_text(url, headers, 'Problems Downloading') parsed_download_url = urlparse(direct_download_url) if parsed_download_url.scheme not in ['http', 'https']: # url is relative, and is missing the scheme and netloc parsed_parent_url = urlparse(url) parsed_download_url = ParseResult(parsed_parent_url.scheme, parsed_parent_url.netloc, parsed_download_url.path, parsed_download_url.params, parsed_download_url.query, parsed_download_url.fragment) direct_download_url = parsed_download_url.geturl() direct_download_url = href_from_link_text(direct_download_url, headers, 'direct link') return direct_download_url
def fetch_resource(gatekeeper_endpoint: ParseResult, workspace_dir: str, kind: Kind, vendor: str, name: str, version: str) -> Tuple[str, Dict[str, Any]]: """Fetch a resource and return the Json as a dictionary. Return `None` if nothing is found. It raise a RuntimeError exception when a gatekeeper API is dectected""" url = urljoin(gatekeeper_endpoint.geturl(), _get_path_from_kind(kind)) _LOGGER.info('Fetching a %s resource by name at %s', kind, url) query_params_raw = {'vendor': vendor, # Dict[Str, Str] 'name': name, 'version': version} # We force the order of the query's parameters to lower the impact on tests # when a key is added or removed query_params = collections.OrderedDict(sorted(query_params_raw.items())) auth = 'Bearer ' + _get_workspace_token(workspace_dir) res_resp = requests.get(url, params=query_params, headers={'content-type': 'application/json', 'Authorization': auth}) try: res_resp.raise_for_status() except requests.exceptions.HTTPError: _LOGGER.exception('Failed to retrieve a resource at %s ' '(status code = %d)', res_resp.url, res_resp.status_code) # REMARK: if nothing is found, then the API return an empty [] and # not 404 raise tmp = res_resp.json() if not isinstance(tmp, list): exc = RuntimeError('The returned json is not boxed by a list') _LOGGER.error('The GK API must return a list of resources: %s', exc) raise exc _LOGGER.info('Succeed to retrieve the resource %s (status code = %d): %s', res_resp.url, res_resp.status_code, tmp[:20]) for elt in tmp: this_uuid = elt['uuid'] if not this_uuid: _LOGGER.warning('Ignoring an element without id or uuid: %s', elt) continue if kind.name in elt: # the resource is boxed elt = elt[kind.name] if all([elt['vendor'] == vendor, elt['name'] == name, elt['version'] == version]): return (this_uuid, elt) return None
def __init__(self, parsed: ParseResult = None): self.parsed = parsed if parsed is None: self.kind = TaskCreationType.Empty self.url = None return path = parsed.path scheme = parsed.scheme url = parsed.geturl() # subtaskInfo is set by Model self.subtaskInfo = [] self.url = url self.path = None self.kind = None if path.endswith(".torrent"): if scheme == "": self.kind = TaskCreationType.LocalTorrent return elif scheme in ("http", "https", "ftp"): self.kind = TaskCreationType.RemoteTorrent return if path.endswith(".metalink") or path.endswith(".meta4"): if scheme in ("http", "https", "ftp"): self.kind = TaskCreationType.MetaLink elif scheme == "ed2k": self.kind = TaskCreationType.Emule elif scheme == "magnet": self.kind = TaskCreationType.Magnet elif scheme in ("http", "https", "ftp"): self.kind = TaskCreationType.Normal
def batch_raw_query(prometheus_endpoint: ParseResult, start_timestamp: int, end_timestamp: int, step: datetime.timedelta, query: str, maxpts=11000) -> Iterable[bytes]: """Retrieve metrics from a Prometheus database""" sstep = '{}s'.format(int(step.total_seconds())) url = urljoin(prometheus_endpoint.geturl(), 'api/v1/query_range') def sub(sub_start, sub_end): """sub""" payload = [('start', sub_start), ('end', sub_end), ('step', sstep), ('query', query)] req = requests.get(url, params=payload) return req.content delta = end_timestamp - start_timestamp batch_size = min(delta // int(step.total_seconds()), maxpts) # type: int for limits in _create_batches(start_timestamp, end_timestamp, batch_size): sub_start, sub_end = limits yield sub(sub_start, sub_end)
class Url(object): """An object based representation of a Url. """ @property def scheme(self): return self._parts.scheme @property def netloc(self): return self._parts.netloc @property def hostname(self): return self._parts.hostname @property def subdomain(self): """ Returns the subdomain for the URL. With thanks: http://stackoverflow.com/questions/1189128/regex-to-extract-subdomain-from-url """ regex = r'(?:http[s]*\:\/\/)*(.*?)\.(?=[^\/]*\..{2,5})' hostname = self.hostname if not hostname: return None matches = re.match(regex, hostname) return matches.group(1) if matches else None @property def port(self): return self._parts.port @property def path(self): return self._parts.path def path_index(self, index=0): try: split_path = self.path.strip('/').split('/') return split_path[index] except: return None @property def params(self): return self._parts.params @property def query(self): return self._parts.query @property def fragment(self): return self._parts.fragment @property def username(self): return self._parts.username @property def password(self): return self._parts.password def __init__(self, url): """Initialize the url object. Create a new Url object from either a well formed url string, a dict of key/values, or a ParseResult. Args: url (mixed): The value to generate the url from. """ if isinstance(url, ParseResult): self._parts = url elif isinstance(url, dict): if 'hostname' in url and 'netloc' not in url: netloc = url.pop('hostname') if 'port' in url: netloc += ':' + str(url.pop('port')) url['netloc'] = netloc if 'scheme' not in url: url['scheme'] = 'http' if 'username' in url: url['netloc'] = '{0}:{1}@{2}'.format( url.pop('username'), url.pop('password', ''), url['netloc']) if 'params' not in url: url['params'] = None if 'fragment' not in url: url['fragment'] = None if 'path' not in url: url['path'] = '/' if 'query' not in url: url['query'] = None self._parts = ParseResult(**url) elif isinstance(url, str): self._parts = urlparse(url) def assemble(self): return self._parts.geturl() def __str__(self): return self.assemble()
def __init__(self, username, password, site='https://tutor-qa.openstax.org', email=None, email_username=None, email_password=None, driver_type='chrome', capabilities=None, pasta_user=None, wait_time=DEFAULT_WAIT_TIME, opera_driver='', existing_driver=None, **kwargs): """ Base user constructor. username (string): website username password (string): website password site (string): website URL driver_type (string): web browser type pasta_user (PastaSauce): optional API access for saucelabs capabilities (dict): browser settings; copy object to avoid overwrite Defaults: DesiredCapabilities.ANDROID.copy() DesiredCapabilities.CHROME.copy() DesiredCapabilities.EDGE.copy() DesiredCapabilities.FIREFOX.copy() DesiredCapabilities.HTMLUNIT.copy() DesiredCapabilities.HTMLUNITWITHJS.copy() DesiredCapabilities.INTERNETEXPLORER.copy() DesiredCapabilities.IPAD.copy() DesiredCapabilities.IPHONE.copy() DesiredCapabilities.ORERA.copy() DesiredCapabilities.PHANTOMJS.copy() DesiredCapabilities.SAFARI.copy() Keys: platform browserName version javascriptEnabled wait (int): standard time, in seconds, to wait for Selenium commands opera_driver (string): Chromium location """ self.username = username self.password = password parse = list( urlparse( site if urlparse(site).scheme else '%s%s' % ('//', site) ) ) parse[0] = b'https' for index, value in enumerate(parse): parse[index] = value.decode('utf-8') if isinstance(value, bytes) \ else value parse = ParseResult(*parse) self.url = parse.geturl() self.email = email self.email_username = email_username self.email_password = email_password super(User, self).__init__(driver_type=driver_type, capabilities=capabilities, pasta_user=pasta_user, wait_time=wait_time, opera_driver=opera_driver, existing_driver=existing_driver, **kwargs)