def testFilterSpec(self):
foo = FilterSpec('*.foo')
self.assertEqual(foo, FilterSpec(foo))
self.assertEqual(foo, FilterSpec(str(foo)))
bar = FilterSpec('https:*.bar')
self.assertEqual(bar, FilterSpec(bar))
self.assertEqual(bar, FilterSpec(str(bar)))
assert foo.match(URL('https://example.foo/blargh'))
assert not bar.match(URL('https://example.foo/blargh'))
assert bar.match(URL('https://example.bar/blargh'))
assert not bar.match(URL('conarys://example.bar/blargh'))
def __init__(self, masterUrl, jobData):
self.masterUrl = URL(masterUrl)
self.imageBase = '%sapi/v1/images/%d' % (masterUrl, jobData['buildId'])
self.uploadBase = '%suploadBuild/%d/' % (masterUrl, jobData['buildId'])
self.outputToken = jobData['outputToken']
self.opener = opener.URLOpener(connectAttempts=2, followRedirects=True)
# Create a logger for things that are inside the log sending path, so
# we can log to console without causing an infinite loop.
self.log = logging.getLogger(__name__ + '.proxy')
self.log.__class__ = NonSendingLogger
def testAddStrategy(self):
m = ProxyMap()
m.addStrategy('example.foo', ['http://proxy1', 'https://proxy2'])
m.addStrategy('example.bar', ['conary://proxy3/conary/'])
m.addStrategy('http:*', ['http://*****:*****@proxy4'],
replaceScheme='conary')
m.addStrategy('https:*', ['https://*****:*****@proxy4'],
replaceScheme='conary')
m.addStrategy('*', ['https://proxy5'])
self.assertEqual(m.items(), [
(FilterSpec(None, Hostname('example.foo')), [
URL('http', (None, None), HostPort('proxy1', 80), ''),
URL('https', (None, None), HostPort('proxy2', 443), ''),
]),
(FilterSpec(None, Hostname('example.bar')), [
URL('conary', (None, None), HostPort('proxy3', 80),
'/conary/')]),
(FilterSpec('http', HostGlob('*')), [
URL('conary', ('user', 'pass'), HostPort('proxy4:80'), '')]),
(FilterSpec('https', HostGlob('*')), [
URL('conarys', ('user', 'pass'), HostPort('proxy4:443'), '')]),
(FilterSpec(None, HostGlob('*')), [
URL('https', (None, None), HostPort('proxy5:443'), '')]),
])
def __init__(self, cfg):
self.cfg = cfg
base = URL(cfg.wmsBase)
self.wmsUser = base.userpass
self.base = base._replace(userpass=None)
self.opener = opener.URLOpener(followRedirects=True)
class ResponseProxy(BaseResponseProxy):
def __init__(self, masterUrl, jobData):
self.masterUrl = URL(masterUrl)
self.imageBase = '%sapi/v1/images/%d' % (masterUrl, jobData['buildId'])
self.uploadBase = '%suploadBuild/%d/' % (masterUrl, jobData['buildId'])
self.outputToken = jobData['outputToken']
self.opener = opener.URLOpener(connectAttempts=2, followRedirects=True)
# Create a logger for things that are inside the log sending path, so
# we can log to console without causing an infinite loop.
self.log = logging.getLogger(__name__ + '.proxy')
self.log.__class__ = NonSendingLogger
def post(self, method, path, contentType='application/xml', body=None):
headers = {
'Content-Type': contentType,
'X-rBuilder-OutputToken': self.outputToken,
}
if path is None:
url = self.imageBase
else:
url = "%s/%s" % (self.imageBase.rstrip('/'), path)
return self.opener.open(url, data=body, method=method, headers=headers)
def postFileObject(self, method, targetName, fobj, size):
headers = {
'Content-Type': 'application/octet-stream',
'X-rBuilder-OutputToken': self.outputToken,
}
url = self.uploadBase + targetName
req = self.opener.newRequest(url, method=method, headers=headers)
body = DigestingReader(fobj)
req.setData(body, size=size)
self.opener.open(req)
return body.hexdigest()
def postFile(self, method, targetName, filePath):
fobj = open(filePath, 'rb')
size = os.fstat(fobj.fileno()).st_size
return self.postFileObject(method, targetName, fobj, size)
def sendStatus(self, code, message):
root = ET.Element('image')
ET.SubElement(root, "status").text = str(code)
ET.SubElement(root, "status_message").text = message
try:
self.post('PUT', path=None, body=ET.tostring(root))
except:
if code >= jobstatus.FINISHED:
# Don't eat errors from sending a final status.
raise
log.exception("Failed to send status upstream")
def sendLog(self, data):
try:
try:
self.post('POST', 'build_log', contentType='text/plain', body=data)
except http_error.ResponseError, err:
if err.errcode != 204: # No Content
raise
except:
self.log.exception("Error sending build log:")
def postOutput(self, fileList, withMetadata=True, attributes=None):
root = ET.Element('files')
for n, (filePath, description) in enumerate(fileList):
# unicodify file names, dropping any invalid bytes
fileName = os.path.basename(filePath).decode('utf8', 'ignore')
fileSize = os.stat(filePath).st_size
log.info("Uploading %d of %d: %s (%d bytes)",
n + 1, len(fileList), fileName, fileSize)
digest = self.postFile('PUT', fileName, filePath)
log.info(" %s uploaded, SHA-1 digest is %s", fileName, digest)
file = ET.SubElement(root, 'file')
ET.SubElement(file, 'title').text = description
ET.SubElement(file, 'size').text = str(fileSize)
ET.SubElement(file, 'sha1').text = digest
ET.SubElement(file, 'file_name').text = fileName
attr = ET.SubElement(root, 'attributes')
if attributes:
for key, value in attributes.iteritems():
ET.SubElement(attr, key).text = str(value)
if withMetadata:
self.post('PUT', 'build_files', body=ET.tostring(root))
def getImage(self, imageUrl):
imageUrl = URL(str(imageUrl))
imageUrl = self.masterUrl.join(imageUrl.path)
return self.opener.open(imageUrl)
def testProxyIter(self):
m = ProxyMap()
m.addStrategy('example.foo', ['http://proxy1', 'https://proxy2'])
m.addStrategy('http:*', ['http://*****:*****@proxy4'],
replaceScheme='conary')
m.addStrategy('https:*', ['https://*****:*****@proxy4'],
replaceScheme='conary')
m.addStrategy('http:*', ['https://proxy5'])
i = m.getProxyIter(URL('http://unrelated.foo'))
self.assertEqual(i.next(), URL('https://proxy5'))
self.assertRaises(StopIteration, i.next)
m.blacklistUrl(URL('https://proxy5'))
i = m.getProxyIter(URL('http://unrelated.foo'))
self.assertRaises(StopIteration, i.next)
i = m.getProxyIter(URL('https://unrelated.foo'))
self.assertEqual(i.next(), DirectConnection)
self.assertRaises(StopIteration, i.next)
i = m.getProxyIter(URL('http://example.foo/bar'))
expected = set([URL('http://proxy1'), URL('https://proxy2')])
while expected:
got = i.next()
assert got in expected
expected.remove(got)
self.assertRaises(StopIteration, i.next)
i = m.getProxyIter(URL('https://example.foo/bar'),
protocolFilter=('http', 'https', 'conary', 'conarys'))
expected = set([URL('http://proxy1'), URL('https://proxy2')])
while expected:
got = i.next()
assert got in expected
expected.remove(got)
self.assertEqual(i.next(), URL('conarys://*****:*****@proxy4'))
self.assertRaises(StopIteration, i.next)
def testUrlJoin(self):
base = URL('https://*****:*****@example.com/one/two?three/four')
self.assertEqual(base.join('foo'),
URL('https://*****:*****@example.com/one/foo'))
self.assertEqual(base.join('./foo'),
URL('https://*****:*****@example.com/one/foo'))
self.assertEqual(base.join('../foo'),
URL('https://*****:*****@example.com/foo'))
self.assertEqual(base.join('../foo?bar=baz/bork'),
URL('https://*****:*****@example.com/foo?bar=baz/bork'))
self.assertEqual(base.join('../../../../../etc/passwd'),
URL('https://*****:*****@example.com/etc/passwd'))
self.assertEqual(base.join('..'),
URL('https://*****:*****@example.com'))
self.assertEqual(base.join('/foo'),
URL('https://*****:*****@example.com/foo'))
self.assertEqual(base.join('/'),
URL('https://*****:*****@example.com/'))
self.assertEqual(base.join('//subdomain.example.com/barbaz'),
URL('https://subdomain.example.com/barbaz'))
self.assertEqual(base.join('//subdomain.example.com'),
URL('https://subdomain.example.com'))
self.assertEqual(base.join('http://fullyqualified.example.com'),
URL('http://fullyqualified.example.com'))
self.assertEqual(base.join('http://fullyqualified.example.com/bork'),
URL('http://fullyqualified.example.com/bork'))
base = URL('https://*****:*****@example.com/')
self.assertEqual(base.join('foo'),
URL('https://*****:*****@example.com/foo'))
self.assertEqual(base.join('../foo'),
URL('https://*****:*****@example.com/foo'))
base = URL('https://*****:*****@example.com')
self.assertEqual(base.join('foo'),
URL('https://*****:*****@example.com/foo'))
def postRpc(self):
if self.request.content_type != 'text/xml':
return self._makeError('400 Bad Request',
"Unrecognized Content-Type")
stream = self.request.body_file
encoding = self.request.headers.get('Content-Encoding', 'identity')
if encoding == 'deflate':
stream = util.decompressStream(stream)
stream.seek(0)
elif encoding != 'identity':
return self._makeError('400 Bad Request',
"Unrecognized Content-Encoding")
try:
params, method = util.xmlrpcLoad(stream)
except:
return self._makeError('400 Bad Request',
"Malformed XMLRPC request")
localAddr = '%s:%s' % (socket.gethostname(), self.getLocalPort())
try:
request = self.requestFilter.fromWire(params)
except (TypeError, ValueError, IndexError):
return self._makeError('400 Bad Request',
"Malformed XMLRPC arguments")
rawUrl = self.request.url
scheme = self.request.headers.get('X-Conary-Proxy-Target-Scheme')
if scheme in ('http', 'https'):
rawUrl = str(URL(rawUrl)._replace(scheme=scheme))
# Execution phase -- locate and call the target method
try:
responseArgs, extraInfo = self.proxyServer.callWrapper(
protocol=None,
port=None,
methodname=method,
authToken=self.auth,
request=request,
remoteIp=self.auth.remote_ip,
rawUrl=rawUrl,
localAddr=localAddr,
protocolString=self.request.http_version,
headers=self.request.headers,
isSecure=self.isSecure)
except errors.InsufficientPermission:
return self._makeError('403 Forbidden', "Insufficient permission")
rawResponse, headers = responseArgs.toWire(request.version)
if extraInfo:
headers['Via'] = proxy.formatViaHeader(localAddr,
self.request.http_version,
prefix=extraInfo.getVia())
response = self.responseFactory(headerlist=headers.items())
response.content_type = 'text/xml'
# Output phase -- serialize and write the response
body = util.xmlrpcDump((rawResponse, ), methodresponse=1)
accept = self.request.accept_encoding
if len(body) > 200 and 'deflate' in accept:
response.content_encoding = 'deflate'
response.body = zlib.compress(body, 5)
else:
response.body = body
if (method == 'getChangeSet' and request.version >= 71
and not responseArgs.isException and response.status_int == 200
and responseArgs.result[0]
and 'multipart/mixed' in list(self.request.accept)):
return self.inlineChangeset(response, responseArgs, headers)
else:
return response
def testUrlJoin(self):
base = URL("https://*****:*****@example.com/one/two?three/four")
self.assertEqual(base.join("foo"), URL("https://*****:*****@example.com/one/foo"))
self.assertEqual(base.join("./foo"), URL("https://*****:*****@example.com/one/foo"))
self.assertEqual(base.join("../foo"), URL("https://*****:*****@example.com/foo"))
self.assertEqual(base.join("../foo?bar=baz/bork"), URL("https://*****:*****@example.com/foo?bar=baz/bork"))
self.assertEqual(base.join("../../../../../etc/passwd"), URL("https://*****:*****@example.com/etc/passwd"))
self.assertEqual(base.join(".."), URL("https://*****:*****@example.com"))
self.assertEqual(base.join("/foo"), URL("https://*****:*****@example.com/foo"))
self.assertEqual(base.join("/"), URL("https://*****:*****@example.com/"))
self.assertEqual(base.join("//subdomain.example.com/barbaz"), URL("https://subdomain.example.com/barbaz"))
self.assertEqual(base.join("//subdomain.example.com"), URL("https://subdomain.example.com"))
self.assertEqual(base.join("http://fullyqualified.example.com"), URL("http://fullyqualified.example.com"))
self.assertEqual(
base.join("http://fullyqualified.example.com/bork"), URL("http://fullyqualified.example.com/bork")
)
base = URL("https://*****:*****@example.com/")
self.assertEqual(base.join("foo"), URL("https://*****:*****@example.com/foo"))
self.assertEqual(base.join("../foo"), URL("https://*****:*****@example.com/foo"))
base = URL("https://*****:*****@example.com")
self.assertEqual(base.join("foo"), URL("https://*****:*****@example.com/foo"))
def __init__(self, cfg):
self.cfg = cfg
base = URL(cfg.wmsBase)
self.wmsUser = base.userpass
self.base = base._replace(userpass=None)
self.opener = opener.URLOpener(followRedirects=True)