Exemple #1
0
 def testFilterSpec(self):
     foo = FilterSpec('*.foo')
     self.assertEqual(foo, FilterSpec(foo))
     self.assertEqual(foo, FilterSpec(str(foo)))
     bar = FilterSpec('https:*.bar')
     self.assertEqual(bar, FilterSpec(bar))
     self.assertEqual(bar, FilterSpec(str(bar)))
     assert foo.match(URL('https://example.foo/blargh'))
     assert not bar.match(URL('https://example.foo/blargh'))
     assert bar.match(URL('https://example.bar/blargh'))
     assert not bar.match(URL('conarys://example.bar/blargh'))
Exemple #2
0
    def __init__(self, masterUrl, jobData):
        self.masterUrl = URL(masterUrl)
        self.imageBase = '%sapi/v1/images/%d' % (masterUrl, jobData['buildId'])
        self.uploadBase = '%suploadBuild/%d/' % (masterUrl, jobData['buildId'])
        self.outputToken = jobData['outputToken']
        self.opener = opener.URLOpener(connectAttempts=2, followRedirects=True)

        # Create a logger for things that are inside the log sending path, so
        # we can log to console without causing an infinite loop.
        self.log = logging.getLogger(__name__ + '.proxy')
        self.log.__class__ = NonSendingLogger
Exemple #3
0
 def testAddStrategy(self):
     m = ProxyMap()
     m.addStrategy('example.foo', ['http://proxy1', 'https://proxy2'])
     m.addStrategy('example.bar', ['conary://proxy3/conary/'])
     m.addStrategy('http:*', ['http://*****:*****@proxy4'],
             replaceScheme='conary')
     m.addStrategy('https:*', ['https://*****:*****@proxy4'],
             replaceScheme='conary')
     m.addStrategy('*', ['https://proxy5'])
     self.assertEqual(m.items(), [
         (FilterSpec(None, Hostname('example.foo')), [
             URL('http', (None, None), HostPort('proxy1', 80), ''),
             URL('https', (None, None), HostPort('proxy2', 443), ''),
             ]),
         (FilterSpec(None, Hostname('example.bar')), [
             URL('conary', (None, None), HostPort('proxy3', 80),
                 '/conary/')]),
         (FilterSpec('http', HostGlob('*')), [
             URL('conary', ('user', 'pass'), HostPort('proxy4:80'), '')]),
         (FilterSpec('https', HostGlob('*')), [
             URL('conarys', ('user', 'pass'), HostPort('proxy4:443'), '')]),
         (FilterSpec(None, HostGlob('*')), [
             URL('https', (None, None), HostPort('proxy5:443'), '')]),
         ])
Exemple #4
0
 def __init__(self, cfg):
     self.cfg = cfg
     base = URL(cfg.wmsBase)
     self.wmsUser = base.userpass
     self.base = base._replace(userpass=None)
     self.opener = opener.URLOpener(followRedirects=True)
Exemple #5
0
class ResponseProxy(BaseResponseProxy):
    def __init__(self, masterUrl, jobData):
        self.masterUrl = URL(masterUrl)
        self.imageBase = '%sapi/v1/images/%d' % (masterUrl, jobData['buildId'])
        self.uploadBase = '%suploadBuild/%d/' % (masterUrl, jobData['buildId'])
        self.outputToken = jobData['outputToken']
        self.opener = opener.URLOpener(connectAttempts=2, followRedirects=True)

        # Create a logger for things that are inside the log sending path, so
        # we can log to console without causing an infinite loop.
        self.log = logging.getLogger(__name__ + '.proxy')
        self.log.__class__ = NonSendingLogger

    def post(self, method, path, contentType='application/xml', body=None):
        headers = {
                'Content-Type': contentType,
                'X-rBuilder-OutputToken': self.outputToken,
                }
        if path is None:
            url = self.imageBase
        else:
            url = "%s/%s" % (self.imageBase.rstrip('/'), path)
        return self.opener.open(url, data=body, method=method, headers=headers)

    def postFileObject(self, method, targetName, fobj, size):
        headers = {
                'Content-Type': 'application/octet-stream',
                'X-rBuilder-OutputToken': self.outputToken,
                }
        url = self.uploadBase + targetName
        req = self.opener.newRequest(url, method=method, headers=headers)
        body = DigestingReader(fobj)
        req.setData(body, size=size)
        self.opener.open(req)
        return body.hexdigest()

    def postFile(self, method, targetName, filePath):
        fobj = open(filePath, 'rb')
        size = os.fstat(fobj.fileno()).st_size
        return self.postFileObject(method, targetName, fobj, size)

    def sendStatus(self, code, message):
        root = ET.Element('image')
        ET.SubElement(root, "status").text = str(code)
        ET.SubElement(root, "status_message").text = message
        try:
            self.post('PUT', path=None, body=ET.tostring(root))
        except:
            if code >= jobstatus.FINISHED:
                # Don't eat errors from sending a final status.
                raise
            log.exception("Failed to send status upstream")

    def sendLog(self, data):
        try:
            try:
                self.post('POST', 'build_log', contentType='text/plain', body=data)
            except http_error.ResponseError, err:
                if err.errcode != 204: # No Content
                    raise
        except:
            self.log.exception("Error sending build log:")

    def postOutput(self, fileList, withMetadata=True, attributes=None):
        root = ET.Element('files')
        for n, (filePath, description) in enumerate(fileList):
            # unicodify file names, dropping any invalid bytes
            fileName = os.path.basename(filePath).decode('utf8', 'ignore')

            fileSize = os.stat(filePath).st_size
            log.info("Uploading %d of %d: %s (%d bytes)",
                    n + 1, len(fileList), fileName, fileSize)

            digest = self.postFile('PUT', fileName, filePath)
            log.info(" %s uploaded, SHA-1 digest is %s", fileName, digest)

            file = ET.SubElement(root, 'file')
            ET.SubElement(file, 'title').text = description
            ET.SubElement(file, 'size').text = str(fileSize)
            ET.SubElement(file, 'sha1').text = digest
            ET.SubElement(file, 'file_name').text = fileName
        attr = ET.SubElement(root, 'attributes')
        if attributes:
            for key, value in attributes.iteritems():
                ET.SubElement(attr, key).text = str(value)

        if withMetadata:
            self.post('PUT', 'build_files', body=ET.tostring(root))

    def getImage(self, imageUrl):
        imageUrl = URL(str(imageUrl))
        imageUrl = self.masterUrl.join(imageUrl.path)
        return self.opener.open(imageUrl)
Exemple #6
0
    def testProxyIter(self):
        m = ProxyMap()
        m.addStrategy('example.foo', ['http://proxy1', 'https://proxy2'])
        m.addStrategy('http:*', ['http://*****:*****@proxy4'],
                replaceScheme='conary')
        m.addStrategy('https:*', ['https://*****:*****@proxy4'],
                replaceScheme='conary')
        m.addStrategy('http:*', ['https://proxy5'])

        i = m.getProxyIter(URL('http://unrelated.foo'))
        self.assertEqual(i.next(), URL('https://proxy5'))
        self.assertRaises(StopIteration, i.next)

        m.blacklistUrl(URL('https://proxy5'))
        i = m.getProxyIter(URL('http://unrelated.foo'))
        self.assertRaises(StopIteration, i.next)

        i = m.getProxyIter(URL('https://unrelated.foo'))
        self.assertEqual(i.next(), DirectConnection)
        self.assertRaises(StopIteration, i.next)

        i = m.getProxyIter(URL('http://example.foo/bar'))
        expected = set([URL('http://proxy1'), URL('https://proxy2')])
        while expected:
            got = i.next()
            assert got in expected
            expected.remove(got)
        self.assertRaises(StopIteration, i.next)

        i = m.getProxyIter(URL('https://example.foo/bar'),
                protocolFilter=('http', 'https', 'conary', 'conarys'))
        expected = set([URL('http://proxy1'), URL('https://proxy2')])
        while expected:
            got = i.next()
            assert got in expected
            expected.remove(got)
        self.assertEqual(i.next(), URL('conarys://*****:*****@proxy4'))
        self.assertRaises(StopIteration, i.next)
Exemple #7
0
    def testUrlJoin(self):
        base = URL('https://*****:*****@example.com/one/two?three/four')
        self.assertEqual(base.join('foo'),
                URL('https://*****:*****@example.com/one/foo'))
        self.assertEqual(base.join('./foo'),
                URL('https://*****:*****@example.com/one/foo'))
        self.assertEqual(base.join('../foo'),
                URL('https://*****:*****@example.com/foo'))
        self.assertEqual(base.join('../foo?bar=baz/bork'),
                URL('https://*****:*****@example.com/foo?bar=baz/bork'))
        self.assertEqual(base.join('../../../../../etc/passwd'),
                URL('https://*****:*****@example.com/etc/passwd'))
        self.assertEqual(base.join('..'),
                URL('https://*****:*****@example.com'))
        self.assertEqual(base.join('/foo'),
                URL('https://*****:*****@example.com/foo'))
        self.assertEqual(base.join('/'),
                URL('https://*****:*****@example.com/'))
        self.assertEqual(base.join('//subdomain.example.com/barbaz'),
                URL('https://subdomain.example.com/barbaz'))
        self.assertEqual(base.join('//subdomain.example.com'),
                URL('https://subdomain.example.com'))
        self.assertEqual(base.join('http://fullyqualified.example.com'),
                URL('http://fullyqualified.example.com'))
        self.assertEqual(base.join('http://fullyqualified.example.com/bork'),
                URL('http://fullyqualified.example.com/bork'))

        base = URL('https://*****:*****@example.com/')
        self.assertEqual(base.join('foo'),
                URL('https://*****:*****@example.com/foo'))
        self.assertEqual(base.join('../foo'),
                URL('https://*****:*****@example.com/foo'))

        base = URL('https://*****:*****@example.com')
        self.assertEqual(base.join('foo'),
                URL('https://*****:*****@example.com/foo'))
Exemple #8
0
    def postRpc(self):
        if self.request.content_type != 'text/xml':
            return self._makeError('400 Bad Request',
                                   "Unrecognized Content-Type")
        stream = self.request.body_file
        encoding = self.request.headers.get('Content-Encoding', 'identity')
        if encoding == 'deflate':
            stream = util.decompressStream(stream)
            stream.seek(0)
        elif encoding != 'identity':
            return self._makeError('400 Bad Request',
                                   "Unrecognized Content-Encoding")

        try:
            params, method = util.xmlrpcLoad(stream)
        except:
            return self._makeError('400 Bad Request',
                                   "Malformed XMLRPC request")

        localAddr = '%s:%s' % (socket.gethostname(), self.getLocalPort())
        try:
            request = self.requestFilter.fromWire(params)
        except (TypeError, ValueError, IndexError):
            return self._makeError('400 Bad Request',
                                   "Malformed XMLRPC arguments")

        rawUrl = self.request.url
        scheme = self.request.headers.get('X-Conary-Proxy-Target-Scheme')
        if scheme in ('http', 'https'):
            rawUrl = str(URL(rawUrl)._replace(scheme=scheme))

        # Execution phase -- locate and call the target method
        try:
            responseArgs, extraInfo = self.proxyServer.callWrapper(
                protocol=None,
                port=None,
                methodname=method,
                authToken=self.auth,
                request=request,
                remoteIp=self.auth.remote_ip,
                rawUrl=rawUrl,
                localAddr=localAddr,
                protocolString=self.request.http_version,
                headers=self.request.headers,
                isSecure=self.isSecure)
        except errors.InsufficientPermission:
            return self._makeError('403 Forbidden', "Insufficient permission")

        rawResponse, headers = responseArgs.toWire(request.version)
        if extraInfo:
            headers['Via'] = proxy.formatViaHeader(localAddr,
                                                   self.request.http_version,
                                                   prefix=extraInfo.getVia())
        response = self.responseFactory(headerlist=headers.items())
        response.content_type = 'text/xml'

        # Output phase -- serialize and write the response
        body = util.xmlrpcDump((rawResponse, ), methodresponse=1)
        accept = self.request.accept_encoding
        if len(body) > 200 and 'deflate' in accept:
            response.content_encoding = 'deflate'
            response.body = zlib.compress(body, 5)
        else:
            response.body = body

        if (method == 'getChangeSet' and request.version >= 71
                and not responseArgs.isException and response.status_int == 200
                and responseArgs.result[0]
                and 'multipart/mixed' in list(self.request.accept)):
            return self.inlineChangeset(response, responseArgs, headers)
        else:
            return response
Exemple #9
0
    def testUrlJoin(self):
        base = URL("https://*****:*****@example.com/one/two?three/four")
        self.assertEqual(base.join("foo"), URL("https://*****:*****@example.com/one/foo"))
        self.assertEqual(base.join("./foo"), URL("https://*****:*****@example.com/one/foo"))
        self.assertEqual(base.join("../foo"), URL("https://*****:*****@example.com/foo"))
        self.assertEqual(base.join("../foo?bar=baz/bork"), URL("https://*****:*****@example.com/foo?bar=baz/bork"))
        self.assertEqual(base.join("../../../../../etc/passwd"), URL("https://*****:*****@example.com/etc/passwd"))
        self.assertEqual(base.join(".."), URL("https://*****:*****@example.com"))
        self.assertEqual(base.join("/foo"), URL("https://*****:*****@example.com/foo"))
        self.assertEqual(base.join("/"), URL("https://*****:*****@example.com/"))
        self.assertEqual(base.join("//subdomain.example.com/barbaz"), URL("https://subdomain.example.com/barbaz"))
        self.assertEqual(base.join("//subdomain.example.com"), URL("https://subdomain.example.com"))
        self.assertEqual(base.join("http://fullyqualified.example.com"), URL("http://fullyqualified.example.com"))
        self.assertEqual(
            base.join("http://fullyqualified.example.com/bork"), URL("http://fullyqualified.example.com/bork")
        )

        base = URL("https://*****:*****@example.com/")
        self.assertEqual(base.join("foo"), URL("https://*****:*****@example.com/foo"))
        self.assertEqual(base.join("../foo"), URL("https://*****:*****@example.com/foo"))

        base = URL("https://*****:*****@example.com")
        self.assertEqual(base.join("foo"), URL("https://*****:*****@example.com/foo"))
Exemple #10
0
 def __init__(self, cfg):
     self.cfg = cfg
     base = URL(cfg.wmsBase)
     self.wmsUser = base.userpass
     self.base = base._replace(userpass=None)
     self.opener = opener.URLOpener(followRedirects=True)