def getProxiedURIT(uriT): tmurl = list(urlsplit(uriT)) if app.proxy is not None: # urlsplit put domain in path for "example.com" tmurl[1] = app.proxy # Set replay host/port if no scheme proxyuri = urlsplit(app.proxy) if proxyuri.scheme != '': tmurl[0] = proxyuri.scheme tmurl[1] = proxyuri.netloc + proxyuri.path return tmurl
def get_user_details(self, response): """Generate username from identity url""" values = super(LiveJournalOpenId, self).get_user_details(response) values['username'] = values.get('username') or \ urlsplit(response.identity_url)\ .netloc.split('.', 1)[0] return values
def authorize(user): if user.orcid is None: proofing_state = current_app.proofing_statedb.get_state_by_eppn(user.eppn, raise_on_missing=False) if not proofing_state: current_app.logger.debug('No proofing state found for user {!s}. Initializing new proofing state.'.format( user)) proofing_state = OrcidProofingState({'eduPersonPrincipalName': user.eppn, 'state': get_unique_hash(), 'nonce': get_unique_hash()}) current_app.proofing_statedb.save(proofing_state) claims_request = ClaimsRequest(userinfo=Claims(id=None)) oidc_args = { 'client_id': current_app.oidc_client.client_id, 'response_type': 'code', 'scope': 'openid', 'claims': claims_request.to_json(), 'redirect_uri': url_for('orcid.authorization_response', _external=True), 'state': proofing_state.state, 'nonce': proofing_state.nonce, } authorization_url = '{}?{}'.format(current_app.oidc_client.authorization_endpoint, urlencode(oidc_args)) current_app.logger.debug('Authorization url: {!s}'.format(authorization_url)) current_app.stats.count(name='authn_request') return redirect(authorization_url) # Orcid already connected to user url = urlappend(current_app.config['DASHBOARD_URL'], 'accountlinking') scheme, netloc, path, query_string, fragment = urlsplit(url) new_query_string = urlencode({'msg': ':ERROR:orc.already_connected'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url)
def _build_url(self): url = list(urlsplit(self._path)) qs = parse_qs(url[3]) qs["count"] = self._count qs["page"] = self._page url[3] = urlencode(qs, doseq=True) return urlunsplit(url)
def get_connection(url, tls=None): """This convenience functions returns a :class:`HTTPConnection` or :class:`HTTPSConnection` based on the information contained in URL. :param url: URL string to create a connection for. Alternatively, passing in the results of :py:func:`urlparse.urlsplit` works as well. :param tls: When the URL scheme is ``https``, this is passed in as the ``tls`` parameter to :class:`HTTPSConnection`. """ if isinstance(url, six.string_types): url = urllib_parse.urlsplit(url, 'http') host = url.netloc or 'localhost' host = host.rsplit(':', 1)[0] port = url.port kwargs = {} if six.PY2: # strict is deprecated on Python3 # https://docs.python.org/3.2/library/http.client.html#httpconnection-objects kwargs['strict'] = True if url.scheme == 'https': conn = HTTPSConnection(host, tls=tls, **kwargs) else: conn = HTTPConnection(host, **kwargs) return conn
def update_archive(thermoml_path=None): """Use RSS feeds to find and download any missing ThermoML XML files from the ThermoML archive. Parameters ---------- thermoml_path : str, optional, default=None If specified, use this path to store ThermoML XML files. If None, use the THERMOML_PATH environment variable. """ if thermoml_path is None: if "THERMOML_PATH" in os.environ: thermoml_path = os.environ["THERMOML_PATH"] else: raise(KeyError("You must either specify thermoml_path or the THERMOML_PATH environment variable.")) for key, url in THERMOML_FEEDS.items(): feed = feedparser.parse(url) for entry in feed["entries"]: link = entry["link"] base_filename = urllib_parse.urlsplit(link).path base_filename = base_filename[1:] # Strip off preceeding backslash character so os.path.join will work filename = os.path.join(thermoml_path, base_filename) make_path(filename) if os.path.exists(filename): print("Already downloaded %s from %s" % (filename, link)) else: print("Fetching %s from %s" % (filename, link)) urllib.request.urlretrieve (link, filename)
def remove_user_from_uri(apps, schema_editor): Formula = apps.get_model('formulas', 'Formula') for formula in Formula.objects.all(): url_bits = urlsplit(formula.uri) # don't do it if it's an ssh formula if 'ssh' in url_bits.scheme: continue if url_bits.username: formula.git_username = url_bits.username if url_bits.port: new_netloc = '{}:{}'.format(url_bits.hostname, url_bits.port) else: new_netloc = url_bits.hostname formula.uri = urljoin(( url_bits.scheme, new_netloc, url_bits.path, url_bits.query, url_bits.fragment, )) formula.save()
def on_headers_complete(parser): # http-parser callback: the HTTP header is complete. This is the point # where we hand off the message to our consumer. Going forward, # on_body() will continue to write chunks of the body to message.body. self = ffi.from_handle(parser.data) self._complete_header_value(b'') m = self._message m.message_type = lib.http_message_type(parser) m.version = '{}.{}'.format(parser.http_major, parser.http_minor) if self._server_side: m.method = _http_methods.get(parser.method, '<unknown>') m.url = _ba2s(self._url) try: m.parsed_url = urlsplit(m.url) except ValueError as e: self._error = HttpError('urlsplit(): {!s}'.format(e)) return 2 # error m.is_upgrade = lib.http_is_upgrade(parser) else: m.status_code = parser.status_code m.should_keep_alive = lib.http_should_keep_alive(parser) m.body = StreamReader(self._update_body_size) # Make the message available. There is no need to call # read_buffer_size_change() here as the changes sum up to 0. self._queue.put_nowait(m, self._header_size) self._header_size = 0 # Return 1 if this is a HEAD request, 0 otherwise. This instructs the # parser whether or not a body follows. if not self._requests: return 0 return 1 if self._requests.pop(0) == 'HEAD' else 0
def validate_url(cls, url): """ Return a boolean indicating whether the URL has a protocol and hostname. If a port is specified, ensure it is an integer. Arguments: url (str): The URL to check. Returns: Boolean indicating whether the URL has a protocol and hostname. """ result = urllib_parse.urlsplit(url) # Check that we have a protocol and hostname if not result.scheme or not result.netloc: return False # Check that the port is an integer try: if result.port is not None: int(result.port) elif result.netloc.endswith(':'): # Valid URLs do not end with colons. return False except ValueError: return False else: return True
def rename(self, new_job_name): """Changes the name of this job :param str new_job_name: new name to assign to this job """ args = { "params": { "newName": new_job_name } } self._api.post(self._api.url + "doRename", args=args) # NOTE: In order to properly support jobs that may contain nested # jobs we have to do some URL manipulations to extrapolate the # REST API endpoint for the parent object to which the cloned # view is to be contained. parts = urllib_parse.urlsplit(self._api.url).path.split("/") parts = [cur_part for cur_part in parts if cur_part.strip()] assert len(parts) >= 2 assert parts[-2] == "job" new_url = urllib_parse.urljoin( self._api.url, "/" + "/".join(parts[:-2])) new_url += "/job/" + new_job_name self._api = self._api.clone(new_url) assert self.name == new_job_name
def collect_hosts(hosts): """ Collect a set of hosts and an optional chroot from a string or a list of strings. """ if isinstance(hosts, list): if hosts[-1].strip().startswith('/'): host_ports, chroot = hosts[:-1], hosts[-1] else: host_ports, chroot = hosts, None else: host_ports, chroot = hosts.partition("/")[::2] host_ports = host_ports.split(",") chroot = "/" + chroot if chroot else None result = [] for host_port in host_ports: # put all complexity of dealing with # IPv4 & IPv6 address:port on the urlsplit res = urllib_parse.urlsplit("xxx://" + host_port) host = res.hostname if host is None: raise ValueError("bad hostname") port = int(res.port) if res.port else 2181 result.append((host.strip(), port)) return result, chroot
def test_url_simple(self): connection = Mock() connection.call.return_value = (None, {'start': 0, 'total_size': 0}) page = Page(connection, '/some-path', None) built_qs = parse_qs(urlsplit(page._build_url()).query) self.assertEqual(built_qs, dict( count=[str(DEFAULT_PAGE_ITEM_COUNT)], page=["1"]))
def get_user_details(self, response): """Generate username from identity url""" values = super(YandexOpenId, self).get_user_details(response) values['username'] = values.get('username') or\ urlsplit(response.identity_url)\ .path.strip('/') values['email'] = values.get('email', '') return values
def __init__(self, url, pool_size=None, tls=None, ehlo_as=None, timeout=None, idle_timeout=None): super(HttpRelay, self).__init__(pool_size) self.url = urllib_parse.urlsplit(url, 'http') self.tls = tls self.ehlo_as = ehlo_as or getfqdn() self.timeout = timeout self.idle_timeout = idle_timeout
def build_absolute_uri(self, location=None): if location is None: return None bits = urlsplit(location) if not (bits.scheme and bits.netloc): location = urljoin(self.prod_url, location) return iri_to_uri(location)
def redirect_action(): # Setup a redirect url to action app root scheme, netloc, path, query_string, fragment = urlsplit(request.url) path = url_for('actions.authn') return_url = urlunsplit((scheme, netloc, path, query_string, fragment)) # TODO: Look in ret to figure out if we need to add a query string with a user message ret = _do_action() return redirect(return_url)
def clean_contributions(self): if self.cleaned_data['contributions']: hostname = urlsplit(self.cleaned_data['contributions']).hostname if not hostname.endswith(amo.VALID_CONTRIBUTION_DOMAINS): raise forms.ValidationError(ugettext( 'URL domain must be one of [%s], or a subdomain.' ) % ', '.join(amo.VALID_CONTRIBUTION_DOMAINS)) return self.cleaned_data['contributions']
def forward(apps, schema_editor): FormulaComponent = apps.get_model('formulas', 'FormulaComponent') Formula = apps.get_model('formulas', 'Formula') formulas = {} for formula in Formula.objects.all(): # Get the username out of the URI if it's a private formula if formula.git_username: parse_res = urlsplit(formula.uri) if '@' in parse_res.netloc: new_netloc = parse_res.netloc.split('@')[-1] formula.uri = urlunsplit(( parse_res.scheme, new_netloc, parse_res.path, parse_res.query, parse_res.fragment )) formula.save() if formula.uri not in formulas: formulas[formula.uri] = formula continue # Otherwise we need to delete the formula and everything associated with it for component in FormulaComponent.objects.filter(formula=formula): for bhfc in component.blueprinthostformulacomponent_set.all(): try: bhfc.component = FormulaComponent.objects.get(sls_path=bhfc.component.sls_path, formula=formulas[formula.uri]) bhfc.save() except FormulaComponent.DoesNotExist: bhfc.component.formula = formulas[formula.uri] bhfc.component.save() component.delete() formula.delete() # re-import all the formulas for formula in Formula.objects.all(): # there's nothing we can do about private repos without having the password :( if not formula.git_username: try: import_formula(formula.id, '') except FormulaTaskException as e: if 'SPECFILE' in e.message: print('Skipping import of formula: {0}'.format(formula.uri)) else: raise else: print('Please manually update this formula via the API: {0}'.format(formula.uri)) # remove the old ones old_formula_dir = os.path.join(settings.STACKDIO_CONFIG['storage_root'], 'user_states') if os.path.isdir(old_formula_dir): shutil.rmtree(old_formula_dir)
def relurl(url, starturl): """Works like :py:func:`os.path.relpath`, but for urls >>> relurl("http://example.org/other/index.html", "http://example.org/main/index.html") == '../other/index.html' True >>> relurl("http://other.org/foo.html", "http://example.org/bar.html") == 'http://other.org/foo.html' True """ urlseg = urlsplit(url) startseg = urlsplit(starturl) urldomain = urlunsplit(urlseg[:2] + tuple('' for i in range(3))) startdomain = urlunsplit(startseg[:2] + tuple('' for i in range(3))) if urldomain != startdomain: # different domain, no relative url possible return url relpath = posixpath.relpath(urlseg.path, posixpath.dirname(startseg.path)) res = urlunsplit(('', '', relpath, urlseg.query, urlseg.fragment)) return res
def test_url_with_qs(self): connection = Mock() connection.call.return_value = (None, {'start': 0, 'total_size': 0}) page = Page(connection, '/some-path?with=a&query=string', None) built_qs = parse_qs(urlsplit(page._build_url()).query) self.assertEqual(built_qs, { "with": ["a"], "query": ["string"], "count": [str(DEFAULT_PAGE_ITEM_COUNT)], "page": ["1"], })
def _generate_helper(self, response_type=None, response_disposition=None, generation=None): endpoint = 'http://api.example.com' resource = '/name/path' credentials = _make_credentials( signing=True, signer_email='*****@*****.**') credentials.sign_bytes.return_value = b'DEADBEEF' signed = base64.b64encode(credentials.sign_bytes.return_value) signed = signed.decode('ascii') expiration = 1000 url = self._call_fut( credentials, resource, expiration, api_access_endpoint=endpoint, response_type=response_type, response_disposition=response_disposition, generation=generation, ) # Check the mock was called. string_to_sign = '\n'.join([ 'GET', '', '', str(expiration), resource, ]) credentials.sign_bytes.assert_called_once_with(string_to_sign) scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'api.example.com') self.assertEqual(path, resource) self.assertEqual(frag, '') # Check the URL parameters. params = urllib_parse.parse_qs(qs) expected_params = { 'GoogleAccessId': [credentials.signer_email], 'Expires': [str(expiration)], 'Signature': [signed], } if response_type is not None: expected_params['response-content-type'] = [response_type] if response_disposition is not None: expected_params['response-content-disposition'] = [ response_disposition] if generation is not None: expected_params['generation'] = [generation] self.assertEqual(params, expected_params)
def clone_to_temp(formula, git_password): # temporary directory to clone into so we can read the # SPECFILE and do some initial validation tmpdir = mkdtemp(prefix='stackdio-') reponame = formula.get_repo_name() repodir = os.path.join(tmpdir, reponame) origin = None repo = None uri = formula.uri # Add the password for a private repo if formula.private_git_repo: parsed = urlsplit(uri) uri = urlunsplit(( parsed.scheme, '{0}:{1}@{2}'.format(formula.git_username, git_password, parsed.netloc), parsed.path, parsed.query, parsed.fragment )) try: # Clone the repo into a temp dir repo = git.Repo.clone_from(uri, repodir) origin = repo.remotes.origin.name except git.GitCommandError: raise FormulaTaskException( formula, 'Unable to clone provided URI. This is either not ' 'a git repository, or you don\'t have permission to clone it. ' 'Note that private repositories require the https protocol.') finally: if repo and not origin: origin = repo.remotes.origin.name if repo and formula and formula.private_git_repo: # remove the password from the config repo.git.remote('set-url', origin, formula.uri) # Remove the logs which also store the password log_dir = os.path.join(repodir, '.git', 'logs') if os.path.isdir(log_dir): shutil.rmtree(log_dir) # return the path where the repo is return repodir
def verify_link(user): """ Used for verifying an e-mail address when the user clicks the link in the verification mail. """ proofing_user = ProofingUser.from_user(user, current_app.private_userdb) code = request.args.get('code') email = request.args.get('email') if code and email: current_app.logger.debug('Trying to save email address {} as verified for user {}'.format(email, proofing_user)) url = urlappend(current_app.config['DASHBOARD_URL'], 'emails') scheme, netloc, path, query_string, fragment = urlsplit(url) try: state = current_app.proofing_statedb.get_state_by_eppn_and_email(proofing_user.eppn, email) timeout = current_app.config.get('EMAIL_VERIFICATION_TIMEOUT', 24) if state.is_expired(timeout): current_app.logger.info("Verification code is expired. Removing the state") current_app.logger.debug("Proofing state: {}".format(state)) current_app.proofing_statedb.remove_state(state) new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) except DocumentDoesNotExist: current_app.logger.info('Could not find proofing state for email {}'.format(email)) new_query_string = urlencode({'msg': ':ERROR:emails.unknown_email'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) if code == state.verification.verification_code: try: verify_mail_address(state, proofing_user) current_app.logger.info('Email successfully verified') current_app.logger.debug('Email address: {}'.format(email)) new_query_string = urlencode({'msg': 'emails.verification-success'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) except UserOutOfSync: current_app.logger.info('Could not confirm email, data out of sync') current_app.logger.debug('Mail address: {}'.format(email)) new_query_string = urlencode({'msg': ':ERROR:user-out-of-sync'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) current_app.logger.info("Invalid verification code") current_app.logger.debug("Email address: {}".format(state.verification.email)) new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) abort(400)
def _SerializeRequest(self, request): """Convert a http_wrapper.Request object into a string. Args: request: A http_wrapper.Request to serialize. Returns: The request as a string in application/http format. """ # Construct status line parsed = urllib_parse.urlsplit(request.url) request_line = urllib_parse.urlunsplit( (None, None, parsed.path, parsed.query, None)) status_line = u' '.join(( request.http_method, request_line.decode('utf-8'), u'HTTP/1.1\n' )) major, minor = request.headers.get( 'content-type', 'application/json').split('/') msg = mime_nonmultipart.MIMENonMultipart(major, minor) # MIMENonMultipart adds its own Content-Type header. # Keep all of the other headers in `request.headers`. for key, value in request.headers.items(): if key == 'content-type': continue msg[key] = value msg['Host'] = parsed.netloc msg.set_unixfrom(None) if request.body is not None: msg.set_payload(request.body) # Serialize the mime message. str_io = six.StringIO() # maxheaderlen=0 means don't line wrap headers. gen = generator.Generator(str_io, maxheaderlen=0) gen.flatten(msg, unixfrom=False) body = str_io.getvalue() # Strip off the \n\n that the MIME lib tacks onto the end of the # payload. if request.body is None: body = body[:-2] return status_line + body
def _generate_helper( self, response_type=None, response_disposition=None, generation=None ): endpoint = "http://api.example.com" resource = "/name/path" credentials = _make_credentials( signing=True, signer_email="*****@*****.**" ) credentials.sign_bytes.return_value = b"DEADBEEF" signed = base64.b64encode(credentials.sign_bytes.return_value) signed = signed.decode("ascii") expiration = 1000 url = self._call_fut( credentials, resource, expiration, api_access_endpoint=endpoint, response_type=response_type, response_disposition=response_disposition, generation=generation, ) # Check the mock was called. string_to_sign = "\n".join(["GET", "", "", str(expiration), resource]) credentials.sign_bytes.assert_called_once_with(string_to_sign) scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url) self.assertEqual(scheme, "http") self.assertEqual(netloc, "api.example.com") self.assertEqual(path, resource) self.assertEqual(frag, "") # Check the URL parameters. params = urllib_parse.parse_qs(qs) expected_params = { "GoogleAccessId": [credentials.signer_email], "Expires": [str(expiration)], "Signature": [signed], } if response_type is not None: expected_params["response-content-type"] = [response_type] if response_disposition is not None: expected_params["response-content-disposition"] = [response_disposition] if generation is not None: expected_params["generation"] = [generation] self.assertEqual(params, expected_params)
def transport_from_url(url): """ Create a transport for the given URL. """ if '/' not in url and ':' in url and url.rsplit(':')[-1].isdigit(): url = 'scgi://' + url url = urlparse.urlsplit(url, scheme="scgi", allow_fragments=False) # pylint: disable=redundant-keyword-arg try: transport = TRANSPORTS[url.scheme.lower()] except KeyError: if not any((url.netloc, url.query)) and url.path.isdigit(): # Support simplified "domain:port" URLs return transport_from_url("scgi://%s:%s" % (url.scheme, url.path)) else: raise URLError("Unsupported scheme in URL %r" % url.geturl()) else: return transport(url)
def url_to_path(url): # type: (str) -> ByteString """ Convert a valid file url to a local filesystem path Follows logic taken from pip's equivalent function """ from .misc import to_bytes assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths if netloc: netloc = "\\\\" + netloc path = urllib_request.url2pathname(netloc + path) return to_bytes(path, encoding="utf-8")
def update_archive(thermoml_path=None): """Use RSS feeds to find and download ThermoML tar files from the ThermoML archive, then download any missing entries by enumerating the RSS feeds. The output will be a flat directory of XML files in `thermoml_path` Parameters ---------- thermoml_path : str, optional, default=None If specified, use this path to store ThermoML XML files. If None, use the THERMOML_PATH environment variable. """ if thermoml_path is None: # Try to obtain the path to the local ThermoML Archive mirror from an environment variable. try: # Check THERMOML_PATH environment variable thermoml_path = os.environ["THERMOML_PATH"] except: # Use default path of ~/.thermoml thermoml_path = os.path.join(os.environ["HOME"], '.thermoml') if not os.path.exists(thermoml_path): os.makedirs(thermoml_path) for key, url in THERMOML_TARBALLS.items(): print("Downloading %s" % url) file = urllib.request.URLopener() local_filename = key + ".tgz" file.retrieve(url, local_filename) tarball = tarfile.open(local_filename) tarball.extractall(thermoml_path) # Update local repository according to feeds. for key, url in THERMOML_FEEDS.items(): print("Fetching RSS %s" % url) feed = feedparser.parse(url) for entry in feed["entries"]: link = entry["link"] base_filename = urllib_parse.urlsplit(link).path base_filename = os.path.split(base_filename)[-1] # Flattens the directory structure to a flat directory to make the tar and RSS files compatible. filename = os.path.join(thermoml_path, base_filename) make_path(filename) if os.path.exists(filename): print("Already downloaded %s from %s" % (filename, link)) else: print("Fetching %s from %s" % (filename, link)) urllib.request.urlretrieve (link, filename)
def collect_hosts(hosts): """Collect a set of hosts and an optional chroot from a string.""" host_ports, chroot = hosts.partition("/")[::2] chroot = "/" + chroot if chroot else None result = [] for host_port in host_ports.split(","): # put all complexity of dealing with # IPv4 & IPv6 address:port on the urlsplit res = urllib_parse.urlsplit("xxx://" + host_port) host = res.hostname if host is None: raise ValueError("bad hostname") port = int(res.port) if res.port else 2181 result.append((host.strip(), port)) return result, chroot
def verify_remote_file_manifest(user, remote_file_manifest): tc = load_transfer_client(user) new_manifest = [] for record in remote_file_manifest: surl = urlsplit(record['url']) if surl.scheme not in settings.SUPPORTED_STAGING_PROTOCOLS: new_manifest.append(record) log.debug('Verification skipped record {} (non-globus)'.format( record['filename'] )) continue globus_endpoint = surl.netloc.replace(':', '') tc.endpoint_autoactivate(globus_endpoint) new_man = _walk_globus_path(tc, globus_endpoint, surl.path) new_manifest += new_man return new_manifest
def __call__(self, value): try: super(URLValidator, self).__call__(value) except ValidationError as e: # Trivial case failed. Try for possible IDN domain if value: # value = force_text(value) scheme, netloc, path, query, fragment = urlsplit(value) try: netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE except UnicodeError: # invalid domain part raise e url = urlunsplit((scheme, netloc, path, query, fragment)) super(URLValidator, self).__call__(url) else: raise else: pass
def validate_remote_files_manifest(remote_files_manifest, transfer_token): transfer_authorizer = globus_sdk.AccessTokenAuthorizer(transfer_token) tc = globus_sdk.TransferClient(authorizer=transfer_authorizer) new_manifest = [] for record in remote_files_manifest: surl = urlsplit(record['url']) if surl.scheme not in settings.SUPPORTED_STAGING_PROTOCOLS: new_manifest.append(record) log.debug('Verification skipped record {} (non-globus)'.format( record['name'])) continue globus_endpoint = surl.netloc.replace(':', '') tc.endpoint_autoactivate(globus_endpoint) new_man = _walk_globus_path(tc, globus_endpoint, surl.path) new_manifest += new_man return new_manifest
def mfa_authentication_action(session_info, user): relay_state = request.form.get('RelayState') current_app.logger.debug('RelayState: {}'.format(relay_state)) redirect_url = None if 'eidas_redirect_urls' in session: redirect_url = session['eidas_redirect_urls'].pop(relay_state, None) if not redirect_url: # With no redirect url just redirect the user to dashboard for a new try to log in # TODO: This will result in a error 400 until we put the authentication in the session current_app.logger.error('Missing redirect url for mfa authentication') return redirect_with_msg(current_app.config['ACTION_URL'], ':ERROR:eidas.no_redirect_url') # We get the mfa authentication views "next" argument as base64 to avoid our request sanitation # to replace all & to & redirect_url = base64.b64decode(redirect_url).decode('utf-8') # TODO: Rename verify_relay_state to verify_redirect_url redirect_url = verify_relay_state(redirect_url) if not is_required_loa(session_info, 'loa3'): return redirect_with_msg(redirect_url, ':ERROR:eidas.authn_context_mismatch') if not is_valid_reauthn(session_info): return redirect_with_msg(redirect_url, ':ERROR:eidas.reauthn_expired') # Check that a verified NIN is equal to the asserted attribute personalIdentityNumber asserted_nin = get_saml_attribute(session_info, 'personalIdentityNumber')[0] user_nin = user.nins.verified.find(asserted_nin) if not user_nin: current_app.logger.error('Asserted NIN not matching user verified nins') current_app.logger.debug('Asserted NIN: {}'.format(asserted_nin)) return redirect_with_msg(redirect_url, ':ERROR:eidas.nin_not_matching') session.mfa_action.success = True session.mfa_action.issuer = session_info['issuer'] session.mfa_action.authn_instant = session_info['authn_info'][0][2] session.mfa_action.authn_context = get_authn_ctx(session_info) # Redirect back to action app but to the redirect-action view resp = redirect_with_msg(redirect_url, 'actions.action-completed') scheme, netloc, path, query_string, fragment = urlsplit(resp.location) new_path = urlappend(path, 'redirect-action') new_url = urlunsplit((scheme, netloc, new_path, query_string, fragment)) current_app.logger.debug(f'Redirecting to: {new_url}') return redirect(new_url)
def add_user_to_uri(apps, schema_editor): Formula = apps.get_model('formulas', 'Formula') for formula in Formula.objects.all(): if not formula.git_username: continue url_bits = urlsplit(formula.uri) # rebuild the uri with the username in it formula.uri = urljoin(( url_bits.scheme, '{}:{}'.format(formula.git_username, url_bits.netloc), url_bits.path, url_bits.query, url_bits.fragment, )) formula.save()
def wrapper(self, *args, **kwargs): if not self.session.user_id: if self.request.method in ("GET", "HEAD"): url = self.get_login_url() if "?" not in url: if urlsplit(url).scheme: # if login url is absolute, make next absolute too next_url = self.request.full_url() else: # next_url = self.request.uri next_url = self.request.full_url() url += "?" + urlencode(dict(next=next_url)) self.redirect(url, permanent=True) return raise web.HTTPError(403) return method(self, *args, **kwargs)
def _make_cookies(base_url, cookies, cj): domain = urllib_parse.urlsplit(base_url).hostname for key in cookies: c = http_cookiejar.Cookie(0, key, str(cookies[key].split(';')[0]), port=None, port_specified=False, domain=domain, domain_specified=True, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={}) cj.set_cookie(c) return cj
def set_query_params(url, **kwargs): """ Set or replace query parameters in a URL. >>> set_query_parameter('http://example.com?foo=bar&biz=baz', foo='stuff') 'http://example.com?foo=stuff&biz=baz' :param url: URL :type url: str :param kwargs: Query parameters :type kwargs: dict :return: Modified URL :rtype: str """ scheme, netloc, path, query_string, fragment = urlsplit(url) query_params = parse_qs(query_string) query_params.populate(kwargs) new_query_string = urlencode(query_params, doseq=True) return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def _load_request_headers(self): deadline = time.time() + self.timeout self.sock.settimeout(self.timeout) line = wire.decode(self.fh.readline()) if len(line) == 0: raise Disconnected('no request', level='debug') m = self.req.match(line) try: self.method = m.group(1) except AttributeError: raise Disconnected('bad request: ' + line) self.uri = m.group(2) self.version = m.group(3) or 'HTTP/1.0' parts = urlsplit(self.uri) self.path = parts.path self.components = tuple(x for x in self.path.split('/') if x) self.query = parts.query log.info('%s\t%s\t%s', self.method, self.path, self.query) while True: now = time.time() if now >= deadline: raise socket.timeout() self.sock.settimeout(deadline - now) line = wire.decode(self.fh.readline()) if len(line) < 2 or '\r\n' != line[-2:]: raise Disconnected('bad header line: ' + line) elif len(line) == 2: # done parsing headers self.sock.settimeout(None) return try: h, v = self.hsplit.split(line[0:-2], maxsplit=1) except ValueError: raise Disconnected('invalid header line: ' + line[0:-2]) self.headers.add(h, v)
def id(self): """Gets the numeric identifier of this queued build Guaranteed to return a valid identifier, even when the queue item this object refers to has been invalidated by Jenkins. :rtype: :class:`int` """ # We could try and pull the item ID from the "id" field of our response # data, but to ensure we always have the ability to return a valid # ID even when this queue item has been cleaned up server-side, we # extrapolate the ID from the URL. # # Queue items are defined by a URL that look something like this: # https://server/queue/item/1234 parts = urllib_parse.urlsplit(self._api.url).path.split("/") parts = [cur_part for cur_part in parts if cur_part.strip()] queue_id = parts[-1] if PY2: queue_id = queue_id.decode("utf-8") assert queue_id.isnumeric() return int(queue_id)
def catalog_transfer_manifest(bagit_bags): """Given a list of bagit bags, return a catalogue of all files organized by endpoint, and the list of files that can't be transferred. Returns tuple: ( catalog_dict, error_catalog_dict ) Example: ( { '<Endpoint UUID>': ['filename1', 'filename2'], '<Endpoint UUID>': ['filename3', 'filename4'] }, { 'unsupported_protocol': ['filename5', 'filename6'] } ) """ endpoint_catalog = {} error_catalog = {} for bag in bagit_bags: for url, size, filename in bag.fetch_entries(): surl = urlsplit(url) if surl.scheme not in settings.SUPPORTED_STAGING_PROTOCOLS: prot_errors = error_catalog.get('unsupported_protocol', []) prot_errors.append(url) error_catalog['unsupported_protocol'] = prot_errors continue globus_endpoint = surl.netloc.replace(':', '') payload = endpoint_catalog.get(globus_endpoint, []) payload.append(surl.path) endpoint_catalog[globus_endpoint] = payload if not endpoint_catalog: raise ConciergeException({ 'error': 'No valid data to transfer', 'error_catalog': error_catalog }) return endpoint_catalog, error_catalog
def _handler(ucr, changes): changed_entries = set() for key in changes.keys(): match = re.match('ucs/web/overview/entries/(admin|service)/([^/]+)/.*', key) if match: changed_entries.add(match.group(2)) changed_entries -= set( ['umc', 'invalid-certificate-list', 'root-certificate', 'ldap-master']) portal_logger.debug('Changed: %r' % changed_entries) if not changed_entries: return lo, pos = get_machine_connection() pos.setDn('cn=entry,cn=portals,cn=univention,%s' % ucr.get('ldap/base')) hostname = '%s.%s' % (ucr.get('hostname'), ucr.get('domainname')) # iterate over all ipv4 and ipv6 addresses and append them to the link local_hosts = [hostname] interfaces = Interfaces(ucr) for idev, iconf in interfaces.all_interfaces: # get ipv4 address of device if iconf.ipv4_address(): local_hosts.append(str(iconf.ipv4_address().ip)) # get ipv6 addresses of device for iname in iconf.ipv6_names: local_hosts.append('[%s]' % (iconf.ipv6_address(iname).ip, )) portal_logger.debug('Local hosts are: %r' % local_hosts) attr_entries = {} for changed_entry in changed_entries: attr_entries[changed_entry] = {} for ucr_key in ucr.keys(): match = re.match('ucs/web/overview/entries/([^/]+)/([^/]+)/(.*)', ucr_key) if not match: continue category = match.group(1) cn = match.group(2) key = match.group(3) value = ucr.get(ucr_key) if cn in attr_entries: portal_logger.debug('Matched %r -> %r' % (ucr_key, value)) entry = attr_entries[cn] entry['name'] = cn if '_links' not in entry: links = [] for host in local_hosts: if host: links.append(_Link(host=host)) entry['_links'] = links if key == 'link': for link in entry['_links']: if value.startswith('http'): link.full = value else: link.path = value elif key == 'port_http': if value: for link in entry['_links'][:]: if link.protocol == 'https': link = copy(link) entry['_links'].append(link) link.protocol = 'http' link.port = value elif key == 'port_https': if value: for link in entry['_links'][:]: if link.protocol == 'http': link = copy(link) entry['_links'].append(link) link.protocol = 'https' link.port = value elif key == 'icon': try: if value.startswith('/univention-management-console'): value = '/univention%s' % value[30:] with open('/var/www/%s' % value, 'rb') as fd: entry['icon'] = b64encode(fd.read()).decode('ASCII') except EnvironmentError: pass elif key == 'label': entry.setdefault('displayName', []) entry['displayName'].append(('en_US', value)) elif key == 'label/de': entry.setdefault('displayName', []) entry['displayName'].append(('de_DE', value)) elif key == 'label/fr': entry.setdefault('displayName', []) entry['displayName'].append(('fr_FR', value)) elif key == 'description': entry.setdefault('description', []) entry['description'].append(('en_US', value)) elif key == 'description/de': entry.setdefault('description', []) entry['description'].append(('de_DE', value)) elif key == 'description/fr': entry.setdefault('description', []) entry['description'].append(('fr_FR', value)) elif key == 'link-target': entry['linkTarget'] = value elif key == 'background-color': entry['backgroundColor'] = value else: portal_logger.info('Don\'t know how to handle UCR key %s' % ucr_key) for cn, attrs in attr_entries.items(): dn = 'cn=%s,%s' % (escape_dn_chars(cn), pos.getDn()) unprocessed_links = attrs.pop('_links', []) my_links = set() no_ports = all(not link.port for link in unprocessed_links) for link in unprocessed_links: if no_ports: if link.protocol == 'http': link.port = '80' elif link.protocol == 'https': link.port = '443' if link: my_links.add(('en_US', str(link))) if not link.protocol: link.protocol = 'http' if link: my_links.add(('en_US', str(link))) link.protocol = 'https' if link: my_links.add(('en_US', str(link))) my_links = list(my_links) portal_logger.debug('Processing %s' % dn) portal_logger.debug('Attrs: %r' % attrs) portal_logger.debug('Links: %r' % my_links) try: obj = init_object('portals/entry', lo, pos, dn) except AttributeError: portal_logger.error( 'The handler is not ready yet. Portal modules are not installed. You may have to set the variables again.' ) return except udm_errors.noObject: portal_logger.debug('DN not found...') if my_links: portal_logger.debug('... creating') attrs['link'] = my_links attrs['activated'] = True try: create_object_if_not_exists('portals/entry', lo, pos, **attrs) except udm_errors.insufficientInformation as exc: portal_logger.info('Cannot create: %s' % exc) try: category_pos = position(ucr.get('ldap/base')) category_pos.setDn('cn=category,cn=portals,cn=univention') category_dn = 'cn=domain-%s,%s' % ( escape_dn_chars(category), category_pos.getDn(), ) portal_logger.debug('Adding entry to %s' % (category_dn, )) obj = init_object('portals/category', lo, category_pos, category_dn) entries = obj['entries'] entries.append(dn) modify_object('portals/category', lo, category_pos, category_dn, entries=entries) except udm_errors.noObject: portal_logger.debug('DN not found...') continue links = obj['link'] portal_logger.debug('Existing links: %r' % links) links = [ _link for _link in links if urlsplit(_link[1]).hostname not in local_hosts ] links.extend(my_links) portal_logger.debug('New links: %r' % links) if not links: portal_logger.debug('Removing DN') remove_object_if_exists('portals/entry', lo, pos, dn) else: portal_logger.debug('Modifying DN') attrs['link'] = links modify_object('portals/entry', lo, pos, dn, **attrs)