def testSourceGoesAway(self):
self.resetRepository()
cfg = self.cfg
self.repos = self.openRepository()
self.resetWork()
self.resetCache()
origDir = os.getcwd()
os.chdir(self.workDir)
self.newpkg('test')
os.chdir('test')
testdir = '/'.join((self.workDir, 'test'))
log.setVerbosity(log.INFO)
self.logFilter.add()
recipeStr = """
class SourceGoesAwayTest(PackageRecipe):
name = 'test'
version = '0'
clearBuildReqs()
def setup(r):
r.addSource('%s1', dir='/foo/')
"""
try:
contentServer = rephelp.HTTPServerController(getRequester())
contentURL = contentServer.url()
curl = contentURL + "file1.txt"
self.writeFile('test.recipe', recipeStr % (curl, ))
self.addfile('test.recipe')
# Test all sorts of cooks, to make sure cooking from the recipe
# does not re-download
self.cookItem(self.repos, cfg, 'test.recipe')
self.commit()
self.cookItem(self.repos, cfg, 'test.recipe')
self.cookItem(self.repos, cfg, 'test')
finally:
contentServer.kill()
os.chdir(origDir)
# Get rid of the cache
self.resetWork()
self.resetCache()
# Server for source file is shut down now
try:
# Get rid of this directory
os.chdir(self.workDir)
self.checkout('test')
os.chdir('test')
# Cook the recipe, should work
self.cookItem(self.repos, cfg, 'test.recipe')
finally:
os.chdir(origDir)
def _testSSLCertCheck(self, keyPair=None):
httpServer = rephelp.HTTPServerController(RequestHandler200,
ssl=keyPair)
try:
caPath = os.path.join(resources.get_archive(),
'ssl-cert-authority.pem')
opener = transport.URLOpener(caCerts=[caPath])
opener.open("https://localhost:%s/someurl" % httpServer.port)
finally:
httpServer.kill()
def testCookies(self):
# CNY-321
try:
contentServer = rephelp.HTTPServerController(cookieRequester())
contentURL = contentServer.url()
name = 'foo.tar.gz'
url = contentURL + '/' + name
cached = lookaside.fetchURL(self.cfg, url, name)
f = open(cached, 'r')
self.assertEqual(f.read(), 'Hello, world!\n')
finally:
contentServer.kill()
def testRefreshNoAutoSource(self):
# CNY-1160
self.resetRepository()
cfg = self.cfg
self.repos = self.openRepository()
self.resetWork()
self.resetCache()
origDir = os.getcwd()
os.chdir(self.workDir)
self.newpkg('test')
os.chdir('test')
testdir = '/'.join((self.workDir, 'test'))
recipeStr = """
class SourceGoesAwayTest(PackageRecipe):
name = 'test'
version = '0'
clearBuildReqs()
def setup(r):
r.addSource('%s1', dir='/foo/')
r.addSource('somefile.txt')
"""
try:
contentServer = rephelp.HTTPServerController(getRequester())
contentURL = contentServer.url()
curl = contentURL + "file1.txt"
self.writeFile('test.recipe', recipeStr % (curl, ))
self.addfile('test.recipe')
self.writeFile('somefile.txt', 'Some lame content\n')
self.addfile('somefile.txt')
expected = [
'+ Trying %s1...' % curl,
'+ Downloading %s1...' % curl,
]
ret, out = self.captureOutput(self.commit)
self.assertEqual(out, '\n'.join(expected) + '\n')
expected.append('warning: somefile.txt is not autosourced and '
'cannot be refreshed')
self.logFilter.add()
self.refresh()
self.logFilter.compare(expected)
finally:
contentServer.kill()
os.chdir(origDir)
def testAuth(self):
# CNY-981
try:
contentServer = rephelp.HTTPServerController(authRequester())
baseUrl = contentServer.url()
# test with user:pass
contentURL = 'http://*****:*****@%s' % httplib.urlsplit(baseUrl)[1]
name = 'foo.tar.gz'
url = contentURL + '/' + name
cached = lookaside.fetchURL(self.cfg, url, name)
f = open(cached, 'r')
self.assertEqual(f.read(), 'Hello, world!\n')
# test with no password given
contentURL = 'http://user@%s' % httplib.urlsplit(baseUrl)[1]
name = 'foo2.tar.gz'
url = contentURL + '/' + name
cached = lookaside.fetchURL(self.cfg, url, name)
f = open(cached, 'r')
self.assertEqual(f.read(), 'Hello, world 2!\n')
# test with no password at all
name = 'foo3.tar.gz'
url = baseUrl + '/' + name
cached = self.logCheck(lookaside.fetchURL, (self.cfg, url, name), [
'error: error downloading http://localhost:[0-9]*//foo3.tar.gz: HTTP Error 401: Unauthorized'
],
regExp=True)
self.assertEqual(cached, None)
# test ftp with user:pass
def fakeOpen(od, req, *args, **kw):
self.req = req
import StringIO
s = 'baz file contents'
r = StringIO.StringIO(s)
r.headers = {'contentLength': len(s)}
return r
import urllib2
self.mock(urllib2.OpenerDirector, 'open', fakeOpen)
url = 'ftp://*****:*****@foo.com/bar/baz.tgz'
name = 'baz.tgz'
cached = lookaside.fetchURL(self.cfg, url, name)
self.assertEqual(url, self.req.get_full_url())
self.assertEqual(open(cached).read(), 'baz file contents')
finally:
contentServer.kill()
def testRefreshNegativeLookaside(self):
recipeStr = """
class RefreshTest(PackageRecipe):
name = 'test'
version = '0'
clearBuildReqs()
def setup(r):
r.addSource('%s1', dir='/foo/')
r.addSource('%s2', dir='/foo/')
"""
self.resetRepository()
cfg = self.cfg
self.repos = self.openRepository()
self.resetWork()
self.resetCache()
try:
contentServer = rephelp.HTTPServerController(getRequester())
contentURL = contentServer.url()
#self.logFilter.add()
os.chdir(self.workDir)
self.newpkg('test')
os.chdir('test')
self.writeFile('test.recipe', recipeStr % (contentURL, contentURL))
self.addfile('test.recipe')
self.logCheck(self.commit, (), [
'. Trying http://localhost:[0-9]*/1\.\.\.',
'. Downloading http://localhost:[0-9]*/1\.\.\.',
'. Trying http://localhost:[0-9]*/2\.\.\.',
'. Downloading http://localhost:[0-9]*/2\.\.\.'
],
regExp=True)
self.cookItem(self.repos, cfg, 'test')
negativePath = os.path.join(self.cacheDir, 'NEGATIVE/test',
'localhost:%d' % contentServer.port,
'2')
util.mkdirChain(os.path.dirname(negativePath))
file(negativePath, "w")
self.logCheck(self.refresh, ('1', ), [
'. Trying http://localhost:[0-9]*/1\.\.\.',
'. Downloading http://localhost:[0-9]*/1\.\.\.',
],
regExp=True)
self.assertFalse(os.path.exists(negativePath))
finally:
contentServer.kill()
def testHTTPProxy(self):
'''Make sure that the lookaside cache can fetch through an http proxy'''
if not os.path.exists(rephelp.HTTPProxy.proxyBinPath):
raise testhelp.SkipTestException(
'testHTTPProxy depends on squid being installed')
class Always200Handler(SimpleHTTPRequestHandler):
def log_message(self, *args, **kw):
pass
def do_GET(self):
response = 'Hello, world!'
self.send_response(200)
self.send_header("Content-type", "text/unknown")
self.send_header("Content-Length", len(response))
self.end_headers()
self.wfile.write(response)
# create the file server
server = rephelp.HTTPServerController(Always200Handler)
proxy = rephelp.HTTPProxy(os.path.join(self.workDir, "http-cache"))
proxyUri = proxy.start()
repos = self.openRepository()
try:
self.cfg.proxy = {'http': 'http://*****:*****@localhost:%d/' % proxy.authPort
}
url2 = 'http://localhost:%d/authProxy.txt' \
% (server.port)
path = lookaside.fetchURL(self.cfg, url2, 'recipename')
self.assertTrue(path)
proxy.stop()
l = open(proxy.accessLog).read()
self.assertTrue(url1 in l)
self.assertTrue(url2 in l)
finally:
proxy.stop()
server.stop()
def newService(self):
logFile = os.path.join(self.workDir, "server.log")
RESTHandler._logFile = logFile
srv = rephelp.HTTPServerController(RESTHandler)
self.services.append(srv)
return srv
def testHTTPProxy(self):
if not os.path.exists(rephelp.HTTPProxy.proxyBinPath):
raise testhelp.SkipTestException(
'testHTTPProxy depends on squid being installed')
h = rephelp.HTTPProxy(os.path.join(self.workDir, "http-cache"))
proxyUri = h.start()
def runTests(opener, port, sslport):
url = 'http://localhost:%d/' % port
sslurl = 'https://localhost:%d' % sslport
logsz0 = h.getAccessLogSize()
fd = opener.open(url)
# Drain the file descriptor
fd.read()
logEntry = h.getAccessLogEntry(logsz0)
opener.close()
# logEntry looks like:
# ['1177451531.683', '115', '127.0.0.1', 'TCP_MISS/200', '18671', 'GET',
# 'http://www.rb.rpath.com', '-', 'DIRECT/172.16.58.31', 'text/html']
self.assertEqual(logEntry[5:7], ['GET', url])
logsz0 = h.getAccessLogSize()
fd = opener.open(sslurl)
# Drain the file descriptor
fd.read()
opener.close()
logEntry = h.getAccessLogEntry(logsz0)
self.assertEqual(logEntry[5:7],
['CONNECT', 'localhost:%d' % sslport])
proxies = {
'http': "http://%s" % proxyUri,
'https': "https://%s" % proxyUri,
}
opener = transport.XMLOpener(proxies=proxies)
class Always200Handler(SimpleHTTPRequestHandler):
def log_message(self, *args, **kw):
pass
def do_GET(self):
response = 'Hello, world!'
self.send_response(200)
self.send_header("Content-type", "text/unknown")
self.send_header("Content-Length", len(response))
self.end_headers()
self.wfile.write(response)
server = rephelp.HTTPServerController(Always200Handler)
secureServer = rephelp.HTTPServerController(Always200Handler, ssl=True)
try:
port = server.port
sslport = secureServer.port
runTests(opener, port, sslport)
# Test that setting the proxy environment variable(s) works
environ = os.environ.copy()
for k, v in proxies.items():
k = k + '_proxy'
environ[k.upper()] = v
self.mock(os, "environ", environ)
opener = transport.XMLOpener()
runTests(opener, port, sslport)
# Point environment variables to some dummy location
environ["HTTP_PROXY"] = "http://foo:1"
environ["HTTPS_PROXY"] = "http://foo:1"
# make sure explicit config overrides env var
opener = transport.XMLOpener(proxies=proxies)
runTests(opener, port, sslport)
# Shut down the proxy
h.stop()
# Adding some timeout, otherwise we see a 503 Service Unavailable
self.sleep(.1)
e = self.assertRaises(socket.error, runTests, opener, port,
sslport)
self.assertEqual(
e.args,
(111,
"Connection refused (via HTTP proxy http://%s)" % proxyUri))
finally:
server.kill()
secureServer.kill()
h.stop()
def testDuplicateBasenames(self):
self.resetCache()
self.resetWork()
self.resetRepository()
repos = self.openRepository()
cfg = self.cfg
repCache = lookaside.RepositoryCache(repos, cfg=cfg)
try:
contentServer = rephelp.HTTPServerController(getRequester())
contentURL = contentServer.url()
origDir = os.getcwd()
os.chdir(self.workDir)
self.newpkg('test')
os.chdir('test')
testdir = '/'.join((self.workDir, 'test'))
log.setVerbosity(log.INFO)
self.logFilter.add()
# first, look for a file that does not exist
assert lookaside.findAll(self.cfg,
repCache,
contentURL + '/404/foo',
'test', (testdir, ),
allowNone=True) is None
# make sure that we got a negative cache entry
assert os.stat(self.cacheDir + '/NEGATIVE/test/404/foo')
# now look for a file that does exist
assert lookaside.findAll(self.cfg,
repCache,
contentURL + '/200/foo',
'test', (testdir, ),
allowNone=True) is not None
# make sure that we got a the cache entry
assert os.stat(self.cacheDir + '/test/200/foo')
# put two different files with the same name name in the cache
fooDir = os.path.join(self.cacheDir, 'test/foo.conary.com/foo/')
os.makedirs(fooDir)
self.writeFile(os.path.join(fooDir, 'theFile'),
'Foo version of the file\n')
barDir = os.path.join(self.cacheDir, 'test/bar.conary.com/bar/')
os.makedirs(barDir)
self.writeFile(os.path.join(barDir, 'theFile'),
'Bar version of the file\n')
# this file shouldn't be found
path = lookaside.findAll(self.cfg,
repCache,
'http://baz.conary.com/foo/theFile',
'test', (testdir, ),
allowNone=True)
self.assertEqual(path, None)
# this file should be found and have the right contents
path = lookaside.findAll(self.cfg,
repCache,
'http://foo.conary.com/foo/theFile',
'test', (testdir, ),
allowNone=True)
f = open(path)
self.assertEqual(f.readline()[0:3], 'Foo')
f.close()
# so should this one
path = lookaside.findAll(self.cfg,
repCache,
'http://bar.conary.com/bar/theFile',
'test', (testdir, ),
allowNone=True)
f = open(path)
self.assertEqual(f.readline()[0:3], 'Bar')
f.close()
finally:
contentServer.kill()
def testRepoCooksDoNotReDownload(self):
# CNY-3221
# Make sure that when cooking from the repository we do not hit the
# server again.
# Create archive
file(os.path.join(self.workDir, "file1"), "w").write("cont\n")
archiveFile = os.path.join(self.workDir, "archive-1.tar.gz")
cmd = ["tar", "zcf", archiveFile, "-C", self.workDir, "file1"]
p = rephelp.subprocess.Popen(cmd)
p.communicate()
self.assertTrue(os.path.exists(archiveFile))
class TarGzRequestor(SimpleHTTPRequestHandler):
requestLog = os.path.join(self.workDir, "request.log")
# shut up!
def log_message(self, *args, **kw):
pass
def do_GET(self):
file(self.requestLog, "a").write(self.path + '\n')
if not self.path.endswith('.tar.gz'):
self.send_response(404)
self.end_headers()
return
content = file(archiveFile).read()
self.send_response(200)
self.send_header('Content-Type', 'application/octet-stream')
self.send_header('Content-Length', str(len(content)))
self.end_headers()
self.wfile.write(content)
recipeStr = """
class ArchiveTest(PackageRecipe):
name = 'archive'
version = '1'
clearBuildReqs()
def setup(r):
r.addArchive('%s/', dir='/foo/')
"""
origDir = os.getcwd()
os.chdir(self.workDir)
self.newpkg('archive')
testdir = '/'.join((self.workDir, 'archive'))
try:
os.chdir(testdir)
contentServer = rephelp.HTTPServerController(TarGzRequestor)
contentUrl = contentServer.url()
self.writeFile('archive.recipe', recipeStr % (contentUrl, ))
self.addfile('archive.recipe')
expected = [
'+ Trying %s/archive-1.tar.bz2...' % contentUrl,
'+ Trying %s/archive-1.tar.gz...' % contentUrl,
'+ Downloading %s/archive-1.tar.gz...' % contentUrl,
]
ret, out = self.captureOutput(self.commit)
self.assertEqual(out, '\n'.join(expected) + '\n')
# Make sure we have the proper files in the request log
self.assertEqual(
[x.strip() for x in file(TarGzRequestor.requestLog)], [
'//archive-1.tar.bz2',
'//archive-1.tar.gz',
])
# Empty the request log file
file(TarGzRequestor.requestLog, "w")
# When cooking from the repository, we should not hit the web
# server again
ret = self.cookFromRepository('archive')
# Make sure we did build something
self.assertEqual(ret[0][1], '/localhost@rpl:linux/1-1-1')
# Request log should be empty
self.assertEqual(
[x.strip() for x in file(TarGzRequestor.requestLog)], [])
finally:
contentServer.kill()
os.chdir(origDir)
def testRefresh(self):
# Using 2?param specifically to verify CNY-3722
recipeStr = """
class RefreshTest(PackageRecipe):
name = 'test'
version = '0'
clearBuildReqs()
def setup(r):
r.addSource('%s1', dir='/foo/')
r.addSource('%s2?param', dir='/foo/')
"""
self.resetRepository()
cfg = self.cfg
self.repos = self.openRepository()
self.resetWork()
self.resetCache()
try:
contentServer = rephelp.HTTPServerController(getRequester())
contentURL = contentServer.url()
#self.logFilter.add()
os.chdir(self.workDir)
self.newpkg('test')
os.chdir('test')
self.writeFile('test.recipe', recipeStr % (contentURL, contentURL))
self.addfile('test.recipe')
self.logCheck(self.commit, (), [
'. Trying http://localhost:[0-9]*/1\.\.\.',
'. Downloading http://localhost:[0-9]*/1\.\.\.',
'. Trying http://localhost:[0-9]*/2\?param\.\.\.',
'. Downloading http://localhost:[0-9]*/2\?param\.\.\.'
],
regExp=True)
self.cookItem(self.repos, cfg, 'test')
self.updatePkg(self.workDir, 'test')
self.verifyFile(self.workDir + '/foo/1', '/1:1\n')
self.verifyFile(self.workDir + '/foo/2?param', '/2?param:1\n')
self.logCheck(self.refresh, ('2*', ), [
'. Trying http://localhost:[0-9]*/2\?param\.\.\.',
'. Downloading http://localhost:[0-9]*/2\?param\.\.\.'
],
regExp=True)
self.cookItem(self.repos, cfg, 'test.recipe')
self.logCheck(self.commit, (),
['. found http://localhost:[0-9]*/1 in repository'],
regExp=True)
self.cookItem(self.repos, cfg, 'test')
self.updatePkg(self.workDir, 'test')
self.verifyFile(self.workDir + '/foo/1', '/1:1\n')
self.verifyFile(self.workDir + '/foo/2?param', '/2?param:2\n')
self.logCheck(self.refresh, (), [
'. Trying http://localhost:[0-9]*/1\.\.\.',
'. Downloading http://localhost:[0-9]*/1\.\.\.',
'. Trying http://localhost:[0-9]*/2\?param\.\.\.',
'. Downloading http://localhost:[0-9]*/2\?param\.\.\.'
],
regExp=True)
self.cookItem(self.repos, cfg, 'test.recipe')
self.commit()
self.cookItem(self.repos, cfg, 'test')
self.updatePkg(self.workDir, 'test')
self.verifyFile(self.workDir + '/foo/1', '/1:2\n')
self.verifyFile(self.workDir + '/foo/2?param', '/2?param:3\n')
finally:
contentServer.kill()