Example #1
0
def getProxiedURIT(uriT):
    tmurl = list(urlsplit(uriT))
    if app.proxy is not None:
        # urlsplit put domain in path for "example.com"
        tmurl[1] = app.proxy  # Set replay host/port if no scheme
        proxyuri = urlsplit(app.proxy)
        if proxyuri.scheme != '':
            tmurl[0] = proxyuri.scheme
            tmurl[1] = proxyuri.netloc + proxyuri.path

    return tmurl
Example #2
0
 def get_user_details(self, response):
     """Generate username from identity url"""
     values = super(LiveJournalOpenId, self).get_user_details(response)
     values['username'] = values.get('username') or \
                          urlsplit(response.identity_url)\
                             .netloc.split('.', 1)[0]
     return values
Example #3
0
def authorize(user):
    if user.orcid is None:
        proofing_state = current_app.proofing_statedb.get_state_by_eppn(user.eppn, raise_on_missing=False)
        if not proofing_state:
            current_app.logger.debug('No proofing state found for user {!s}. Initializing new proofing state.'.format(
                user))
            proofing_state = OrcidProofingState({'eduPersonPrincipalName': user.eppn, 'state': get_unique_hash(),
                                                 'nonce': get_unique_hash()})
            current_app.proofing_statedb.save(proofing_state)

        claims_request = ClaimsRequest(userinfo=Claims(id=None))
        oidc_args = {
            'client_id': current_app.oidc_client.client_id,
            'response_type': 'code',
            'scope': 'openid',
            'claims': claims_request.to_json(),
            'redirect_uri': url_for('orcid.authorization_response', _external=True),
            'state': proofing_state.state,
            'nonce': proofing_state.nonce,
        }
        authorization_url = '{}?{}'.format(current_app.oidc_client.authorization_endpoint, urlencode(oidc_args))
        current_app.logger.debug('Authorization url: {!s}'.format(authorization_url))
        current_app.stats.count(name='authn_request')
        return redirect(authorization_url)
    # Orcid already connected to user
    url = urlappend(current_app.config['DASHBOARD_URL'], 'accountlinking')
    scheme, netloc, path, query_string, fragment = urlsplit(url)
    new_query_string = urlencode({'msg': ':ERROR:orc.already_connected'})
    url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
    return redirect(url)
Example #4
0
 def _build_url(self):
     url = list(urlsplit(self._path))
     qs = parse_qs(url[3])
     qs["count"] = self._count
     qs["page"] = self._page
     url[3] = urlencode(qs, doseq=True)
     return urlunsplit(url)
Example #5
0
def get_connection(url, tls=None):
    """This convenience functions returns a :class:`HTTPConnection` or
    :class:`HTTPSConnection` based on the information contained in URL.

    :param url: URL string to create a connection for. Alternatively, passing
                in the results of :py:func:`urlparse.urlsplit` works as well.
    :param tls: When the URL scheme is ``https``, this is passed in as the
                ``tls`` parameter to :class:`HTTPSConnection`.

    """
    if isinstance(url, six.string_types):
        url = urllib_parse.urlsplit(url, 'http')
    host = url.netloc or 'localhost'
    host = host.rsplit(':', 1)[0]
    port = url.port
    kwargs = {}

    if six.PY2:
        # strict is deprecated on Python3
        # https://docs.python.org/3.2/library/http.client.html#httpconnection-objects
        kwargs['strict'] = True

    if url.scheme == 'https':
        conn = HTTPSConnection(host, tls=tls, **kwargs)
    else:
        conn = HTTPConnection(host, **kwargs)
    return conn
Example #6
0
def update_archive(thermoml_path=None):
    """Use RSS feeds to find and download any missing ThermoML XML files
    from the ThermoML archive.  
    
    Parameters
    ----------
    thermoml_path : str, optional, default=None
        If specified, use this path to store ThermoML XML files.  If None,
        use the THERMOML_PATH environment variable.
    """
    if thermoml_path is None:
        if "THERMOML_PATH" in os.environ:
            thermoml_path = os.environ["THERMOML_PATH"]
        else:
            raise(KeyError("You must either specify thermoml_path or the THERMOML_PATH environment variable."))

    for key, url in THERMOML_FEEDS.items():
        feed = feedparser.parse(url)
        for entry in feed["entries"]:
            link = entry["link"]
            base_filename = urllib_parse.urlsplit(link).path
            base_filename = base_filename[1:]  # Strip off preceeding backslash character so os.path.join will work
            filename = os.path.join(thermoml_path, base_filename)
            make_path(filename)
            if os.path.exists(filename):
                print("Already downloaded %s from %s" % (filename, link))
            else:
                print("Fetching %s from %s" % (filename, link))
                urllib.request.urlretrieve (link, filename)
def remove_user_from_uri(apps, schema_editor):
    Formula = apps.get_model('formulas', 'Formula')

    for formula in Formula.objects.all():
        url_bits = urlsplit(formula.uri)

        # don't do it if it's an ssh formula
        if 'ssh' in url_bits.scheme:
            continue

        if url_bits.username:
            formula.git_username = url_bits.username

            if url_bits.port:
                new_netloc = '{}:{}'.format(url_bits.hostname, url_bits.port)
            else:
                new_netloc = url_bits.hostname

            formula.uri = urljoin((
                url_bits.scheme,
                new_netloc,
                url_bits.path,
                url_bits.query,
                url_bits.fragment,
            ))

            formula.save()
Example #8
0
 def on_headers_complete(parser):
     # http-parser callback: the HTTP header is complete. This is the point
     # where we hand off the message to our consumer. Going forward,
     # on_body() will continue to write chunks of the body to message.body.
     self = ffi.from_handle(parser.data)
     self._complete_header_value(b'')
     m = self._message
     m.message_type = lib.http_message_type(parser)
     m.version = '{}.{}'.format(parser.http_major, parser.http_minor)
     if self._server_side:
         m.method = _http_methods.get(parser.method, '<unknown>')
         m.url = _ba2s(self._url)
         try:
             m.parsed_url = urlsplit(m.url)
         except ValueError as e:
             self._error = HttpError('urlsplit(): {!s}'.format(e))
             return 2  # error
         m.is_upgrade = lib.http_is_upgrade(parser)
     else:
         m.status_code = parser.status_code
     m.should_keep_alive = lib.http_should_keep_alive(parser)
     m.body = StreamReader(self._update_body_size)
     # Make the message available. There is no need to call
     # read_buffer_size_change() here as the changes sum up to 0.
     self._queue.put_nowait(m, self._header_size)
     self._header_size = 0
     # Return 1 if this is a HEAD request, 0 otherwise. This instructs the
     # parser whether or not a body follows.
     if not self._requests:
         return 0
     return 1 if self._requests.pop(0) == 'HEAD' else 0
Example #9
0
    def validate_url(cls, url):
        """
        Return a boolean indicating whether the URL has a protocol and hostname.
        If a port is specified, ensure it is an integer.

        Arguments:
            url (str): The URL to check.

        Returns:
            Boolean indicating whether the URL has a protocol and hostname.
        """
        result = urllib_parse.urlsplit(url)

        # Check that we have a protocol and hostname
        if not result.scheme or not result.netloc:
            return False

        # Check that the port is an integer
        try:
            if result.port is not None:
                int(result.port)
            elif result.netloc.endswith(':'):
                # Valid URLs do not end with colons.
                return False
        except ValueError:
            return False
        else:
            return True
Example #10
0
    def rename(self, new_job_name):
        """Changes the name of this job

        :param str new_job_name:
            new name to assign to this job
        """
        args = {
            "params": {
                "newName": new_job_name
            }
        }
        self._api.post(self._api.url + "doRename", args=args)

        # NOTE: In order to properly support jobs that may contain nested
        #       jobs we have to do some URL manipulations to extrapolate the
        #       REST API endpoint for the parent object to which the cloned
        #       view is to be contained.
        parts = urllib_parse.urlsplit(self._api.url).path.split("/")
        parts = [cur_part for cur_part in parts if cur_part.strip()]
        assert len(parts) >= 2
        assert parts[-2] == "job"
        new_url = urllib_parse.urljoin(
            self._api.url, "/" + "/".join(parts[:-2]))
        new_url += "/job/" + new_job_name
        self._api = self._api.clone(new_url)

        assert self.name == new_job_name
Example #11
0
def collect_hosts(hosts):
    """
       Collect a set of hosts and an optional chroot from
       a string or a list of strings.
    """
    if isinstance(hosts, list):
        if hosts[-1].strip().startswith('/'):
            host_ports, chroot = hosts[:-1], hosts[-1]
        else:
            host_ports, chroot = hosts, None
    else:
        host_ports, chroot = hosts.partition("/")[::2]
        host_ports = host_ports.split(",")
        chroot = "/" + chroot if chroot else None

    result = []
    for host_port in host_ports:
        # put all complexity of dealing with
        # IPv4 & IPv6 address:port on the urlsplit
        res = urllib_parse.urlsplit("xxx://" + host_port)
        host = res.hostname
        if host is None:
            raise ValueError("bad hostname")
        port = int(res.port) if res.port else 2181
        result.append((host.strip(), port))

    return result, chroot
Example #12
0
 def test_url_simple(self):
     connection = Mock()
     connection.call.return_value = (None, {'start': 0, 'total_size': 0})
     page = Page(connection, '/some-path', None)
     built_qs = parse_qs(urlsplit(page._build_url()).query)
     self.assertEqual(built_qs, dict(
         count=[str(DEFAULT_PAGE_ITEM_COUNT)],
         page=["1"]))
Example #13
0
 def get_user_details(self, response):
     """Generate username from identity url"""
     values = super(YandexOpenId, self).get_user_details(response)
     values['username'] = values.get('username') or\
                          urlsplit(response.identity_url)\
                                 .path.strip('/')
     values['email'] = values.get('email', '')
     return values
Example #14
0
 def __init__(self, url, pool_size=None, tls=None, ehlo_as=None,
              timeout=None, idle_timeout=None):
     super(HttpRelay, self).__init__(pool_size)
     self.url = urllib_parse.urlsplit(url, 'http')
     self.tls = tls
     self.ehlo_as = ehlo_as or getfqdn()
     self.timeout = timeout
     self.idle_timeout = idle_timeout
Example #15
0
    def build_absolute_uri(self, location=None):
        if location is None:
            return None

        bits = urlsplit(location)
        if not (bits.scheme and bits.netloc):
            location = urljoin(self.prod_url, location)
        return iri_to_uri(location)
Example #16
0
def redirect_action():
    # Setup a redirect url to action app root
    scheme, netloc, path, query_string, fragment = urlsplit(request.url)
    path = url_for('actions.authn')
    return_url = urlunsplit((scheme, netloc, path, query_string, fragment))
    # TODO: Look in ret to figure out if we need to add a query string with a user message
    ret = _do_action()
    return redirect(return_url)
Example #17
0
 def clean_contributions(self):
     if self.cleaned_data['contributions']:
         hostname = urlsplit(self.cleaned_data['contributions']).hostname
         if not hostname.endswith(amo.VALID_CONTRIBUTION_DOMAINS):
             raise forms.ValidationError(ugettext(
                 'URL domain must be one of [%s], or a subdomain.'
             ) % ', '.join(amo.VALID_CONTRIBUTION_DOMAINS))
     return self.cleaned_data['contributions']
def forward(apps, schema_editor):
    FormulaComponent = apps.get_model('formulas', 'FormulaComponent')
    Formula = apps.get_model('formulas', 'Formula')

    formulas = {}

    for formula in Formula.objects.all():
        # Get the username out of the URI if it's a private formula
        if formula.git_username:
            parse_res = urlsplit(formula.uri)
            if '@' in parse_res.netloc:
                new_netloc = parse_res.netloc.split('@')[-1]
                formula.uri = urlunsplit((
                    parse_res.scheme,
                    new_netloc,
                    parse_res.path,
                    parse_res.query,
                    parse_res.fragment
                ))
                formula.save()

        if formula.uri not in formulas:
            formulas[formula.uri] = formula
            continue

        # Otherwise we need to delete the formula and everything associated with it
        for component in FormulaComponent.objects.filter(formula=formula):
            for bhfc in component.blueprinthostformulacomponent_set.all():
                try:
                    bhfc.component = FormulaComponent.objects.get(sls_path=bhfc.component.sls_path,
                                                                  formula=formulas[formula.uri])
                    bhfc.save()
                except FormulaComponent.DoesNotExist:
                    bhfc.component.formula = formulas[formula.uri]
                    bhfc.component.save()

            component.delete()

        formula.delete()

    # re-import all the formulas
    for formula in Formula.objects.all():
        # there's nothing we can do about private repos without having the password :(
        if not formula.git_username:
            try:
                import_formula(formula.id, '')
            except FormulaTaskException as e:
                if 'SPECFILE' in e.message:
                    print('Skipping import of formula: {0}'.format(formula.uri))
                else:
                    raise
        else:
            print('Please manually update this formula via the API: {0}'.format(formula.uri))

    # remove the old ones
    old_formula_dir = os.path.join(settings.STACKDIO_CONFIG['storage_root'], 'user_states')
    if os.path.isdir(old_formula_dir):
        shutil.rmtree(old_formula_dir)
Example #19
0
def relurl(url, starturl):
    """Works like :py:func:`os.path.relpath`, but for urls

    >>> relurl("http://example.org/other/index.html", "http://example.org/main/index.html") == '../other/index.html'
    True
    >>> relurl("http://other.org/foo.html", "http://example.org/bar.html") == 'http://other.org/foo.html'
    True

    """
    urlseg = urlsplit(url)
    startseg = urlsplit(starturl)
    urldomain = urlunsplit(urlseg[:2] + tuple('' for i in range(3)))
    startdomain = urlunsplit(startseg[:2] + tuple('' for i in range(3)))
    if urldomain != startdomain:  # different domain, no relative url possible
        return url

    relpath = posixpath.relpath(urlseg.path, posixpath.dirname(startseg.path))
    res = urlunsplit(('', '', relpath, urlseg.query, urlseg.fragment))
    return res
Example #20
0
 def test_url_with_qs(self):
     connection = Mock()
     connection.call.return_value = (None, {'start': 0, 'total_size': 0})
     page = Page(connection, '/some-path?with=a&query=string', None)
     built_qs = parse_qs(urlsplit(page._build_url()).query)
     self.assertEqual(built_qs, {
         "with": ["a"],
         "query": ["string"],
         "count": [str(DEFAULT_PAGE_ITEM_COUNT)],
         "page": ["1"],
         })
    def _generate_helper(self, response_type=None, response_disposition=None,
                         generation=None):
        endpoint = 'http://api.example.com'
        resource = '/name/path'
        credentials = _make_credentials(
            signing=True, signer_email='*****@*****.**')
        credentials.sign_bytes.return_value = b'DEADBEEF'
        signed = base64.b64encode(credentials.sign_bytes.return_value)
        signed = signed.decode('ascii')

        expiration = 1000
        url = self._call_fut(
            credentials,
            resource,
            expiration,
            api_access_endpoint=endpoint,
            response_type=response_type,
            response_disposition=response_disposition,
            generation=generation,
        )

        # Check the mock was called.
        string_to_sign = '\n'.join([
            'GET',
            '',
            '',
            str(expiration),
            resource,
        ])
        credentials.sign_bytes.assert_called_once_with(string_to_sign)

        scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'api.example.com')
        self.assertEqual(path, resource)
        self.assertEqual(frag, '')

        # Check the URL parameters.
        params = urllib_parse.parse_qs(qs)
        expected_params = {
            'GoogleAccessId': [credentials.signer_email],
            'Expires': [str(expiration)],
            'Signature': [signed],
        }
        if response_type is not None:
            expected_params['response-content-type'] = [response_type]
        if response_disposition is not None:
            expected_params['response-content-disposition'] = [
                response_disposition]
        if generation is not None:
            expected_params['generation'] = [generation]
        self.assertEqual(params, expected_params)
Example #22
0
def clone_to_temp(formula, git_password):
    # temporary directory to clone into so we can read the
    # SPECFILE and do some initial validation
    tmpdir = mkdtemp(prefix='stackdio-')
    reponame = formula.get_repo_name()
    repodir = os.path.join(tmpdir, reponame)
    origin = None
    repo = None

    uri = formula.uri
    # Add the password for a private repo
    if formula.private_git_repo:
        parsed = urlsplit(uri)
        uri = urlunsplit((
            parsed.scheme,
            '{0}:{1}@{2}'.format(formula.git_username, git_password, parsed.netloc),
            parsed.path,
            parsed.query,
            parsed.fragment
        ))

    try:
        # Clone the repo into a temp dir
        repo = git.Repo.clone_from(uri, repodir)

        origin = repo.remotes.origin.name

    except git.GitCommandError:
        raise FormulaTaskException(
            formula,
            'Unable to clone provided URI. This is either not '
            'a git repository, or you don\'t have permission to clone it.  '
            'Note that private repositories require the https protocol.')
    finally:
        if repo and not origin:
            origin = repo.remotes.origin.name

        if repo and formula and formula.private_git_repo:
            # remove the password from the config
            repo.git.remote('set-url', origin, formula.uri)

            # Remove the logs which also store the password
            log_dir = os.path.join(repodir, '.git', 'logs')
            if os.path.isdir(log_dir):
                shutil.rmtree(log_dir)

    # return the path where the repo is
    return repodir
Example #23
0
def verify_link(user):
    """
    Used for verifying an e-mail address when the user clicks the link in the verification mail.
    """
    proofing_user = ProofingUser.from_user(user, current_app.private_userdb)
    code = request.args.get('code')
    email = request.args.get('email')
    if code and email:
        current_app.logger.debug('Trying to save email address {} as verified for user {}'.format(email, proofing_user))
        url = urlappend(current_app.config['DASHBOARD_URL'], 'emails')
        scheme, netloc, path, query_string, fragment = urlsplit(url)

        try:
            state = current_app.proofing_statedb.get_state_by_eppn_and_email(proofing_user.eppn, email)
            timeout = current_app.config.get('EMAIL_VERIFICATION_TIMEOUT', 24)
            if state.is_expired(timeout):
                current_app.logger.info("Verification code is expired. Removing the state")
                current_app.logger.debug("Proofing state: {}".format(state))
                current_app.proofing_statedb.remove_state(state)
                new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'})
                url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
                return redirect(url)
        except DocumentDoesNotExist:
            current_app.logger.info('Could not find proofing state for email {}'.format(email))
            new_query_string = urlencode({'msg': ':ERROR:emails.unknown_email'})
            url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
            return redirect(url)

        if code == state.verification.verification_code:
            try:
                verify_mail_address(state, proofing_user)
                current_app.logger.info('Email successfully verified')
                current_app.logger.debug('Email address: {}'.format(email))
                new_query_string = urlencode({'msg': 'emails.verification-success'})
                url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
                return redirect(url)
            except UserOutOfSync:
                current_app.logger.info('Could not confirm email, data out of sync')
                current_app.logger.debug('Mail address: {}'.format(email))
                new_query_string = urlencode({'msg': ':ERROR:user-out-of-sync'})
                url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
                return redirect(url)
        current_app.logger.info("Invalid verification code")
        current_app.logger.debug("Email address: {}".format(state.verification.email))
        new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'})
        url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
        return redirect(url)
    abort(400)
Example #24
0
    def _SerializeRequest(self, request):
        """Convert a http_wrapper.Request object into a string.

        Args:
          request: A http_wrapper.Request to serialize.

        Returns:
          The request as a string in application/http format.
        """
        # Construct status line
        parsed = urllib_parse.urlsplit(request.url)
        request_line = urllib_parse.urlunsplit(
            (None, None, parsed.path, parsed.query, None))
        status_line = u' '.join((
            request.http_method,
            request_line.decode('utf-8'),
            u'HTTP/1.1\n'
        ))
        major, minor = request.headers.get(
            'content-type', 'application/json').split('/')
        msg = mime_nonmultipart.MIMENonMultipart(major, minor)

        # MIMENonMultipart adds its own Content-Type header.
        # Keep all of the other headers in `request.headers`.
        for key, value in request.headers.items():
            if key == 'content-type':
                continue
            msg[key] = value

        msg['Host'] = parsed.netloc
        msg.set_unixfrom(None)

        if request.body is not None:
            msg.set_payload(request.body)

        # Serialize the mime message.
        str_io = six.StringIO()
        # maxheaderlen=0 means don't line wrap headers.
        gen = generator.Generator(str_io, maxheaderlen=0)
        gen.flatten(msg, unixfrom=False)
        body = str_io.getvalue()

        # Strip off the \n\n that the MIME lib tacks onto the end of the
        # payload.
        if request.body is None:
            body = body[:-2]

        return status_line + body
Example #25
0
    def _generate_helper(
        self, response_type=None, response_disposition=None, generation=None
    ):
        endpoint = "http://api.example.com"
        resource = "/name/path"
        credentials = _make_credentials(
            signing=True, signer_email="*****@*****.**"
        )
        credentials.sign_bytes.return_value = b"DEADBEEF"
        signed = base64.b64encode(credentials.sign_bytes.return_value)
        signed = signed.decode("ascii")

        expiration = 1000
        url = self._call_fut(
            credentials,
            resource,
            expiration,
            api_access_endpoint=endpoint,
            response_type=response_type,
            response_disposition=response_disposition,
            generation=generation,
        )

        # Check the mock was called.
        string_to_sign = "\n".join(["GET", "", "", str(expiration), resource])
        credentials.sign_bytes.assert_called_once_with(string_to_sign)

        scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url)
        self.assertEqual(scheme, "http")
        self.assertEqual(netloc, "api.example.com")
        self.assertEqual(path, resource)
        self.assertEqual(frag, "")

        # Check the URL parameters.
        params = urllib_parse.parse_qs(qs)
        expected_params = {
            "GoogleAccessId": [credentials.signer_email],
            "Expires": [str(expiration)],
            "Signature": [signed],
        }
        if response_type is not None:
            expected_params["response-content-type"] = [response_type]
        if response_disposition is not None:
            expected_params["response-content-disposition"] = [response_disposition]
        if generation is not None:
            expected_params["generation"] = [generation]
        self.assertEqual(params, expected_params)
Example #26
0
def transport_from_url(url):
    """ Create a transport for the given URL.
    """
    if '/' not in url and ':' in url and url.rsplit(':')[-1].isdigit():
        url = 'scgi://' + url
    url = urlparse.urlsplit(url, scheme="scgi", allow_fragments=False)  # pylint: disable=redundant-keyword-arg

    try:
        transport = TRANSPORTS[url.scheme.lower()]
    except KeyError:
        if not any((url.netloc, url.query)) and url.path.isdigit():
            # Support simplified "domain:port" URLs
            return transport_from_url("scgi://%s:%s" % (url.scheme, url.path))
        else:
            raise URLError("Unsupported scheme in URL %r" % url.geturl())
    else:
        return transport(url)
Example #27
0
def transport_from_url(url):
    """ Create a transport for the given URL.
    """
    if '/' not in url and ':' in url and url.rsplit(':')[-1].isdigit():
        url = 'scgi://' + url
    url = urlparse.urlsplit(url, scheme="scgi", allow_fragments=False)  # pylint: disable=redundant-keyword-arg

    try:
        transport = TRANSPORTS[url.scheme.lower()]
    except KeyError:
        if not any((url.netloc, url.query)) and url.path.isdigit():
            # Support simplified "domain:port" URLs
            return transport_from_url("scgi://%s:%s" % (url.scheme, url.path))
        else:
            raise URLError("Unsupported scheme in URL %r" % url.geturl())
    else:
        return transport(url)
Example #28
0
def url_to_path(url):
    # type: (str) -> ByteString
    """
    Convert a valid file url to a local filesystem path

    Follows logic taken from pip's equivalent function
    """
    from .misc import to_bytes

    assert is_file_url(url), "Only file: urls can be converted to local paths"
    _, netloc, path, _, _ = urllib_parse.urlsplit(url)
    # Netlocs are UNC paths
    if netloc:
        netloc = "\\\\" + netloc

    path = urllib_request.url2pathname(netloc + path)
    return to_bytes(path, encoding="utf-8")
Example #29
0
def update_archive(thermoml_path=None):
    """Use RSS feeds to find and download ThermoML tar files
    from the ThermoML archive, then download any missing entries by enumerating the
    RSS feeds.  The output will be a flat directory of XML files in `thermoml_path`

    Parameters
    ----------
    thermoml_path : str, optional, default=None
        If specified, use this path to store ThermoML XML files.  If None,
        use the THERMOML_PATH environment variable.
    """
    if thermoml_path is None:
        # Try to obtain the path to the local ThermoML Archive mirror from an environment variable.
        try:
            # Check THERMOML_PATH environment variable
            thermoml_path = os.environ["THERMOML_PATH"]
        except:
            # Use default path of ~/.thermoml
            thermoml_path = os.path.join(os.environ["HOME"], '.thermoml')

    if not os.path.exists(thermoml_path):
        os.makedirs(thermoml_path)

    for key, url in THERMOML_TARBALLS.items():
        print("Downloading %s" % url)
        file = urllib.request.URLopener()
        local_filename = key + ".tgz"
        file.retrieve(url, local_filename)
        tarball = tarfile.open(local_filename)
        tarball.extractall(thermoml_path)

    # Update local repository according to feeds.
    for key, url in THERMOML_FEEDS.items():
        print("Fetching RSS %s" % url)
        feed = feedparser.parse(url)
        for entry in feed["entries"]:
            link = entry["link"]
            base_filename = urllib_parse.urlsplit(link).path
            base_filename = os.path.split(base_filename)[-1]  # Flattens the directory structure to a flat directory to make the tar and RSS files compatible.
            filename = os.path.join(thermoml_path, base_filename)
            make_path(filename)
            if os.path.exists(filename):
                print("Already downloaded %s from %s" % (filename, link))
            else:
                print("Fetching %s from %s" % (filename, link))
                urllib.request.urlretrieve (link, filename)
Example #30
0
def collect_hosts(hosts):
    """Collect a set of hosts and an optional chroot from a string."""
    host_ports, chroot = hosts.partition("/")[::2]
    chroot = "/" + chroot if chroot else None

    result = []
    for host_port in host_ports.split(","):
        # put all complexity of dealing with
        # IPv4 & IPv6 address:port on the urlsplit
        res = urllib_parse.urlsplit("xxx://" + host_port)
        host = res.hostname
        if host is None:
            raise ValueError("bad hostname")
        port = int(res.port) if res.port else 2181
        result.append((host.strip(), port))

    return result, chroot
Example #31
0
def verify_remote_file_manifest(user, remote_file_manifest):
    tc = load_transfer_client(user)

    new_manifest = []
    for record in remote_file_manifest:
        surl = urlsplit(record['url'])
        if surl.scheme not in settings.SUPPORTED_STAGING_PROTOCOLS:
            new_manifest.append(record)
            log.debug('Verification skipped record {} (non-globus)'.format(
                record['filename']
            ))
            continue
        globus_endpoint = surl.netloc.replace(':', '')
        tc.endpoint_autoactivate(globus_endpoint)
        new_man = _walk_globus_path(tc, globus_endpoint, surl.path)
        new_manifest += new_man

    return new_manifest
Example #32
0
 def __call__(self, value):
     try:
         super(URLValidator, self).__call__(value)
     except ValidationError as e:
         # Trivial case failed. Try for possible IDN domain
         if value:
             # value = force_text(value)
             scheme, netloc, path, query, fragment = urlsplit(value)
             try:
                 netloc = netloc.encode('idna').decode('ascii')  # IDN -> ACE
             except UnicodeError:  # invalid domain part
                 raise e
             url = urlunsplit((scheme, netloc, path, query, fragment))
             super(URLValidator, self).__call__(url)
         else:
             raise
     else:
         pass
Example #33
0
def validate_remote_files_manifest(remote_files_manifest, transfer_token):
    transfer_authorizer = globus_sdk.AccessTokenAuthorizer(transfer_token)
    tc = globus_sdk.TransferClient(authorizer=transfer_authorizer)

    new_manifest = []
    for record in remote_files_manifest:
        surl = urlsplit(record['url'])
        if surl.scheme not in settings.SUPPORTED_STAGING_PROTOCOLS:
            new_manifest.append(record)
            log.debug('Verification skipped record {} (non-globus)'.format(
                record['name']))
            continue
        globus_endpoint = surl.netloc.replace(':', '')
        tc.endpoint_autoactivate(globus_endpoint)
        new_man = _walk_globus_path(tc, globus_endpoint, surl.path)
        new_manifest += new_man

    return new_manifest
Example #34
0
def mfa_authentication_action(session_info, user):
    relay_state = request.form.get('RelayState')
    current_app.logger.debug('RelayState: {}'.format(relay_state))
    redirect_url = None
    if 'eidas_redirect_urls' in session:
        redirect_url = session['eidas_redirect_urls'].pop(relay_state, None)
    if not redirect_url:
        # With no redirect url just redirect the user to dashboard for a new try to log in
        # TODO: This will result in a error 400 until we put the authentication in the session
        current_app.logger.error('Missing redirect url for mfa authentication')
        return redirect_with_msg(current_app.config['ACTION_URL'], ':ERROR:eidas.no_redirect_url')

    # We get the mfa authentication views "next" argument as base64 to avoid our request sanitation
    # to replace all & to &amp;
    redirect_url = base64.b64decode(redirect_url).decode('utf-8')
    # TODO: Rename verify_relay_state to verify_redirect_url
    redirect_url = verify_relay_state(redirect_url)

    if not is_required_loa(session_info, 'loa3'):
        return redirect_with_msg(redirect_url, ':ERROR:eidas.authn_context_mismatch')

    if not is_valid_reauthn(session_info):
        return redirect_with_msg(redirect_url, ':ERROR:eidas.reauthn_expired')

    # Check that a verified NIN is equal to the asserted attribute personalIdentityNumber
    asserted_nin = get_saml_attribute(session_info, 'personalIdentityNumber')[0]
    user_nin = user.nins.verified.find(asserted_nin)
    if not user_nin:
        current_app.logger.error('Asserted NIN not matching user verified nins')
        current_app.logger.debug('Asserted NIN: {}'.format(asserted_nin))
        return redirect_with_msg(redirect_url, ':ERROR:eidas.nin_not_matching')

    session.mfa_action.success = True
    session.mfa_action.issuer = session_info['issuer']
    session.mfa_action.authn_instant = session_info['authn_info'][0][2]
    session.mfa_action.authn_context = get_authn_ctx(session_info)

    # Redirect back to action app but to the redirect-action view
    resp = redirect_with_msg(redirect_url, 'actions.action-completed')
    scheme, netloc, path, query_string, fragment = urlsplit(resp.location)
    new_path = urlappend(path, 'redirect-action')
    new_url = urlunsplit((scheme, netloc, new_path, query_string, fragment))
    current_app.logger.debug(f'Redirecting to: {new_url}')
    return redirect(new_url)
Example #35
0
def add_user_to_uri(apps, schema_editor):
    Formula = apps.get_model('formulas', 'Formula')

    for formula in Formula.objects.all():
        if not formula.git_username:
            continue

        url_bits = urlsplit(formula.uri)

        # rebuild the uri with the username in it
        formula.uri = urljoin((
            url_bits.scheme,
            '{}:{}'.format(formula.git_username, url_bits.netloc),
            url_bits.path,
            url_bits.query,
            url_bits.fragment,
        ))

        formula.save()
Example #36
0
    def wrapper(self, *args, **kwargs):
        if not self.session.user_id:

            if self.request.method in ("GET", "HEAD"):
                url = self.get_login_url()

                if "?" not in url:
                    if urlsplit(url).scheme:
                        # if login url is absolute, make next absolute too
                        next_url = self.request.full_url()

                    else:
                        # next_url = self.request.uri
                        next_url = self.request.full_url()

                    url += "?" + urlencode(dict(next=next_url))
                self.redirect(url, permanent=True)
                return
            raise web.HTTPError(403)
        return method(self, *args, **kwargs)
Example #37
0
def _make_cookies(base_url, cookies, cj):
    domain = urllib_parse.urlsplit(base_url).hostname
    for key in cookies:
        c = http_cookiejar.Cookie(0,
                                  key,
                                  str(cookies[key].split(';')[0]),
                                  port=None,
                                  port_specified=False,
                                  domain=domain,
                                  domain_specified=True,
                                  domain_initial_dot=False,
                                  path='/',
                                  path_specified=True,
                                  secure=False,
                                  expires=None,
                                  discard=False,
                                  comment=None,
                                  comment_url=None,
                                  rest={})
        cj.set_cookie(c)
    return cj
Example #38
0
def set_query_params(url, **kwargs):
    """
    Set or replace query parameters in a URL.

    >>> set_query_parameter('http://example.com?foo=bar&biz=baz', foo='stuff')
    'http://example.com?foo=stuff&biz=baz'

    :param url: URL
    :type url: str
    :param kwargs: Query parameters
    :type kwargs: dict
    :return: Modified URL
    :rtype: str
    """

    scheme, netloc, path, query_string, fragment = urlsplit(url)
    query_params = parse_qs(query_string)

    query_params.populate(kwargs)
    new_query_string = urlencode(query_params, doseq=True)

    return urlunsplit((scheme, netloc, path, new_query_string, fragment))
Example #39
0
    def _load_request_headers(self):
        deadline = time.time() + self.timeout
        self.sock.settimeout(self.timeout)
        line = wire.decode(self.fh.readline())
        if len(line) == 0:
            raise Disconnected('no request', level='debug')

        m = self.req.match(line)
        try:
            self.method = m.group(1)
        except AttributeError:
            raise Disconnected('bad request: ' + line)
        self.uri = m.group(2)
        self.version = m.group(3) or 'HTTP/1.0'
        parts = urlsplit(self.uri)
        self.path = parts.path
        self.components = tuple(x for x in self.path.split('/') if x)
        self.query = parts.query

        log.info('%s\t%s\t%s', self.method, self.path, self.query)

        while True:
            now = time.time()
            if now >= deadline:
                raise socket.timeout()
            self.sock.settimeout(deadline - now)
            line = wire.decode(self.fh.readline())
            if len(line) < 2 or '\r\n' != line[-2:]:
                raise Disconnected('bad header line: ' + line)
            elif len(line) == 2:
                # done parsing headers
                self.sock.settimeout(None)
                return
            try:
                h, v = self.hsplit.split(line[0:-2], maxsplit=1)
            except ValueError:
                raise Disconnected('invalid header line: ' + line[0:-2])
            self.headers.add(h, v)
Example #40
0
    def id(self):
        """Gets the numeric identifier of this queued build

        Guaranteed to return a valid identifier, even when the queue item
        this object refers to has been invalidated by Jenkins.

        :rtype: :class:`int`
        """
        # We could try and pull the item ID from the "id" field of our response
        # data, but to ensure we always have the ability to return a valid
        # ID even when this queue item has been cleaned up server-side, we
        # extrapolate the ID from the URL.
        #
        # Queue items are defined by a URL that look something like this:
        #       https://server/queue/item/1234
        parts = urllib_parse.urlsplit(self._api.url).path.split("/")
        parts = [cur_part for cur_part in parts if cur_part.strip()]
        queue_id = parts[-1]
        if PY2:
            queue_id = queue_id.decode("utf-8")
        assert queue_id.isnumeric()

        return int(queue_id)
Example #41
0
def catalog_transfer_manifest(bagit_bags):
    """Given a list of bagit bags, return a catalogue of all files
    organized by endpoint, and the list of files that can't be
    transferred.

    Returns tuple: ( catalog_dict, error_catalog_dict )
    Example: (
        {
            '<Endpoint UUID>': ['filename1', 'filename2'],
            '<Endpoint UUID>': ['filename3', 'filename4']
        },
        {
            'unsupported_protocol': ['filename5', 'filename6']
        }
    )
    """
    endpoint_catalog = {}
    error_catalog = {}
    for bag in bagit_bags:
        for url, size, filename in bag.fetch_entries():
            surl = urlsplit(url)
            if surl.scheme not in settings.SUPPORTED_STAGING_PROTOCOLS:
                prot_errors = error_catalog.get('unsupported_protocol', [])
                prot_errors.append(url)
                error_catalog['unsupported_protocol'] = prot_errors
                continue
            globus_endpoint = surl.netloc.replace(':', '')
            payload = endpoint_catalog.get(globus_endpoint, [])
            payload.append(surl.path)
            endpoint_catalog[globus_endpoint] = payload
    if not endpoint_catalog:
        raise ConciergeException({
            'error': 'No valid data to transfer',
            'error_catalog': error_catalog
        })
    return endpoint_catalog, error_catalog
Example #42
0
def _handler(ucr, changes):
    changed_entries = set()
    for key in changes.keys():
        match = re.match('ucs/web/overview/entries/(admin|service)/([^/]+)/.*',
                         key)
        if match:
            changed_entries.add(match.group(2))
    changed_entries -= set(
        ['umc', 'invalid-certificate-list', 'root-certificate', 'ldap-master'])
    portal_logger.debug('Changed: %r' % changed_entries)
    if not changed_entries:
        return
    lo, pos = get_machine_connection()
    pos.setDn('cn=entry,cn=portals,cn=univention,%s' % ucr.get('ldap/base'))
    hostname = '%s.%s' % (ucr.get('hostname'), ucr.get('domainname'))

    # iterate over all ipv4 and ipv6 addresses and append them to the link
    local_hosts = [hostname]
    interfaces = Interfaces(ucr)
    for idev, iconf in interfaces.all_interfaces:
        # get ipv4 address of device
        if iconf.ipv4_address():
            local_hosts.append(str(iconf.ipv4_address().ip))

        # get ipv6 addresses of device
        for iname in iconf.ipv6_names:
            local_hosts.append('[%s]' % (iconf.ipv6_address(iname).ip, ))

    portal_logger.debug('Local hosts are: %r' % local_hosts)
    attr_entries = {}
    for changed_entry in changed_entries:
        attr_entries[changed_entry] = {}
    for ucr_key in ucr.keys():
        match = re.match('ucs/web/overview/entries/([^/]+)/([^/]+)/(.*)',
                         ucr_key)
        if not match:
            continue
        category = match.group(1)
        cn = match.group(2)
        key = match.group(3)
        value = ucr.get(ucr_key)
        if cn in attr_entries:
            portal_logger.debug('Matched %r -> %r' % (ucr_key, value))
            entry = attr_entries[cn]
            entry['name'] = cn
            if '_links' not in entry:
                links = []
                for host in local_hosts:
                    if host:
                        links.append(_Link(host=host))
                entry['_links'] = links
            if key == 'link':
                for link in entry['_links']:
                    if value.startswith('http'):
                        link.full = value
                    else:
                        link.path = value
            elif key == 'port_http':
                if value:
                    for link in entry['_links'][:]:
                        if link.protocol == 'https':
                            link = copy(link)
                            entry['_links'].append(link)
                        link.protocol = 'http'
                        link.port = value
            elif key == 'port_https':
                if value:
                    for link in entry['_links'][:]:
                        if link.protocol == 'http':
                            link = copy(link)
                            entry['_links'].append(link)
                        link.protocol = 'https'
                        link.port = value
            elif key == 'icon':
                try:
                    if value.startswith('/univention-management-console'):
                        value = '/univention%s' % value[30:]
                    with open('/var/www/%s' % value, 'rb') as fd:
                        entry['icon'] = b64encode(fd.read()).decode('ASCII')
                except EnvironmentError:
                    pass
            elif key == 'label':
                entry.setdefault('displayName', [])
                entry['displayName'].append(('en_US', value))
            elif key == 'label/de':
                entry.setdefault('displayName', [])
                entry['displayName'].append(('de_DE', value))
            elif key == 'label/fr':
                entry.setdefault('displayName', [])
                entry['displayName'].append(('fr_FR', value))
            elif key == 'description':
                entry.setdefault('description', [])
                entry['description'].append(('en_US', value))
            elif key == 'description/de':
                entry.setdefault('description', [])
                entry['description'].append(('de_DE', value))
            elif key == 'description/fr':
                entry.setdefault('description', [])
                entry['description'].append(('fr_FR', value))
            elif key == 'link-target':
                entry['linkTarget'] = value
            elif key == 'background-color':
                entry['backgroundColor'] = value
            else:
                portal_logger.info('Don\'t know how to handle UCR key %s' %
                                   ucr_key)
    for cn, attrs in attr_entries.items():
        dn = 'cn=%s,%s' % (escape_dn_chars(cn), pos.getDn())
        unprocessed_links = attrs.pop('_links', [])
        my_links = set()
        no_ports = all(not link.port for link in unprocessed_links)
        for link in unprocessed_links:
            if no_ports:
                if link.protocol == 'http':
                    link.port = '80'
                elif link.protocol == 'https':
                    link.port = '443'
            if link:
                my_links.add(('en_US', str(link)))
            if not link.protocol:
                link.protocol = 'http'
                if link:
                    my_links.add(('en_US', str(link)))
                link.protocol = 'https'
                if link:
                    my_links.add(('en_US', str(link)))
        my_links = list(my_links)
        portal_logger.debug('Processing %s' % dn)
        portal_logger.debug('Attrs: %r' % attrs)
        portal_logger.debug('Links: %r' % my_links)
        try:
            obj = init_object('portals/entry', lo, pos, dn)
        except AttributeError:
            portal_logger.error(
                'The handler is not ready yet. Portal modules are not installed. You may have to set the variables again.'
            )
            return
        except udm_errors.noObject:
            portal_logger.debug('DN not found...')
            if my_links:
                portal_logger.debug('... creating')
                attrs['link'] = my_links
                attrs['activated'] = True
                try:
                    create_object_if_not_exists('portals/entry', lo, pos,
                                                **attrs)
                except udm_errors.insufficientInformation as exc:
                    portal_logger.info('Cannot create: %s' % exc)
                try:
                    category_pos = position(ucr.get('ldap/base'))
                    category_pos.setDn('cn=category,cn=portals,cn=univention')
                    category_dn = 'cn=domain-%s,%s' % (
                        escape_dn_chars(category),
                        category_pos.getDn(),
                    )
                    portal_logger.debug('Adding entry to %s' % (category_dn, ))
                    obj = init_object('portals/category', lo, category_pos,
                                      category_dn)
                    entries = obj['entries']
                    entries.append(dn)
                    modify_object('portals/category',
                                  lo,
                                  category_pos,
                                  category_dn,
                                  entries=entries)
                except udm_errors.noObject:
                    portal_logger.debug('DN not found...')
            continue
        links = obj['link']
        portal_logger.debug('Existing links: %r' % links)
        links = [
            _link for _link in links
            if urlsplit(_link[1]).hostname not in local_hosts
        ]
        links.extend(my_links)
        portal_logger.debug('New links: %r' % links)
        if not links:
            portal_logger.debug('Removing DN')
            remove_object_if_exists('portals/entry', lo, pos, dn)
        else:
            portal_logger.debug('Modifying DN')
            attrs['link'] = links
            modify_object('portals/entry', lo, pos, dn, **attrs)