def __init__(self, stateName, tests=None, fastMode=False):
_IntegrationState.__init__(self,
"solr-" + stateName,
tests=tests,
fastMode=fastMode)
self.testdataDir = join(dirname(mydir), 'data/integration')
self.solrStatePath = join(self.integrationTempdir, 'solr')
self.solrPort = PortNumberGenerator.next()
self.solrClientPort = PortNumberGenerator.next()
self.solrCore = "records"
self.config = {
self.solrCore: {
'autocomplete': True,
'suggestions': {
'field': '__all__'
},
'autoCommit': {
'autoCommitMaxTime': 500
}
},
'core2': {},
'core3': {}
}
self.configPath = join(self.integrationTempdir, 'solr.config')
with open(self.configPath, 'w') as f:
f.write(dumps(self.config))
def testHttpGetViaProxy(self):
get_request = []
port = PortNumberGenerator.next()
proxyPort = PortNumberGenerator.next()
self.proxyServer(proxyPort, get_request)
self.referenceHttpServer(port, get_request)
responses = []
def gethandler(*args, **kwargs):
response = yield httpget('localhost', port, '/path',
headers={'Content-Type': 'text/plain', 'Content-Length': 0},
proxyServer="http://localhost:%s" % proxyPort
)
yield response
responses.append(response)
self.handler = gethandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertTrue("GET RESPONSE" in responses[0], responses[0])
self.assertEquals('CONNECT', get_request[0]['command'])
self.assertEquals('localhost:%s' % port, get_request[0]['path'])
self.assertEquals('GET', get_request[1]['command'])
self.assertEquals('/path', get_request[1]['path'])
def testUnbindPortNumberV6(self):
if not has_dual_stack():
return printDualStackSkipped()
p = PortNumberGenerator.next(bind=True)
self.assertNotBound(
bindV4(ip='::1', port=p, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=p)
self.assertBoundAndUnbind(
bindV6(ip='::1', port=p, protocol='tcp', reuse=False)
) # Same IP-version, protocol, host, port combination: basically testing bindV6's close happened.
for protocol in ['tcp', 'udp']:
self.assertBoundAndUnbind(
bindV4(ip='127.0.0.1', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(
bindV4(ip='0.0.0.0', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(
bindV6(ip='::1', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(
bindV6(ip='::', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(
bindV6(ip='::1',
port=p,
protocol=protocol,
reuse=False,
ipV6Only=False))
self.assertBoundAndUnbind(
bindV6(ip='::',
port=p,
protocol=protocol,
reuse=False,
ipV6Only=False))
def testHttpGetViaProxy(self):
get_request = []
port = PortNumberGenerator.next()
proxyPort = PortNumberGenerator.next()
self.proxyServer(proxyPort, get_request)
self.referenceHttpServer(port, get_request)
responses = []
def gethandler(*args, **kwargs):
response = yield httpget('localhost',
port,
'/path',
headers={
'Content-Type': 'text/plain',
'Content-Length': 0
},
proxyServer="http://localhost:%s" %
proxyPort)
yield response
responses.append(response)
self.handler = gethandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertTrue("GET RESPONSE" in responses[0], responses[0])
self.assertEquals('CONNECT', get_request[0]['command'])
self.assertEquals('localhost:%s' % port, get_request[0]['path'])
self.assertEquals('GET', get_request[1]['command'])
self.assertEquals('/path', get_request[1]['path'])
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self, "examples-" + stateName, tests=tests, fastMode=fastMode)
self.testdataDir = join(dirname(mydir), 'data')
self.gatewayPort = PortNumberGenerator.next()
self.indexPort = PortNumberGenerator.next()
self.apiPort = PortNumberGenerator.next()
self.lucenePort = PortNumberGenerator.next()
def testConnectFails(self):
def failingserver(*args, **kwarg):
response = yield httpget(*target)
self.handler = failingserver
clientget('localhost', self.port, '/')
target = ('localhost', 'port', '/') # non-numeric port
self._loopReactorUntilDone()
expectedTraceback = ignoreLineNumbers("""Traceback (most recent call last):
File "%(__file__)s", line 0, in handle
yield self.handler(*args, **kwargs)
File "%(__file__)s", line 85, in failingserver
response = yield httpget(*target)
File "%(httprequest.py)s", line 78, in httprequest
result = s.getResult()
File "%(suspend.py)s", line 34, in __call__
self._doNext(self)
File "%(httprequest.py)s", line 35, in _do
sok.connect((host, port))
File "<string>", line 1, in connect
TypeError: an integer is required
""" % fileDict)
if PYVERSION == "2.7":
expectedTraceback = ignoreLineNumbers("""Traceback (most recent call last):
File "%(__file__)s", line 0, in handle
yield self.handler(*args, **kwargs)
File "%(__file__)s", line 85, in failingserver
response = yield httpget(*target)
File "%(httprequest.py)s", line 78, in httprequest
result = s.getResult()
File "%(suspend.py)s", line 34, in __call__
self._doNext(self)
File "%(httprequest.py)s", line 35, in _do
sok.connect((host, port))
File "/usr/lib/python2.7/socket.py", line [#], in meth
return getattr(self._sock,name)(*args)
TypeError: an integer is required
""" % fileDict)
self.assertEquals(TypeError, self.error[0])
self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(''.join(format_exception(*self.error))))
target = ('localhost', 87, '/') # invalid port
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertEquals(IOError, self.error[0])
target = ('UEYR^$*FD(#>NDJ.khfd9.(*njnd', PortNumberGenerator.next(), '/') # invalid host
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertEquals(SocketGaiError, self.error[0])
target = ('127.0.0.1', PortNumberGenerator.next(), '/') # No-one listens
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertEquals(IOError, self.error[0])
self.assertEquals('111', str(self.error[1]))
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self,
stateName,
tests=tests,
fastMode=fastMode)
self.testdataDir = join(dirname(mydir), 'updateRequest')
self.gatewayPort = PortNumberGenerator.next()
self.apiPort = PortNumberGenerator.next()
self.resolverPort = PortNumberGenerator.next()
def testBindPortRangeNumberGiven(self):
port = PortNumberGenerator.next(blockSize=2)
port2 = port + 1
PortNumberGenerator.bind(port=port, blockSize=2)
self.assertNotBound(bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
self.assertNotBound(bindV4(ip='127.0.0.1', port=port2, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=port, blockSize=2)
self.assertBoundAndUnbind(bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
self.assertBoundAndUnbind(bindV4(ip='127.0.0.1', port=port2, protocol='tcp', reuse=True))
def testUnbindPortNumberV4(self):
p = PortNumberGenerator.next(bind=True)
self.assertNotBound(bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=p)
self.assertBoundAndUnbind(bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=False))
self.assertBoundAndUnbind(bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=False)) # Same IP-version, protocol, host, port combination: basically testing bindV4's close happened.
self.assertBoundAndUnbind(bindV4(ip='127.0.0.1', port=p, protocol='udp', reuse=False))
self.assertBoundAndUnbind(bindV4(ip='0.0.0.0', port=p, protocol='tcp', reuse=False))
self.assertBoundAndUnbind(bindV4(ip='0.0.0.0', port=p, protocol='udp', reuse=False))
def __init__(self, stateName, tests=None, fastMode=False):
SeecrIntegrationState.__init__(self, "meresco-lucene-" + stateName, tests=tests, fastMode=fastMode)
self.stateName = stateName
if not fastMode:
system('rm -rf ' + self.integrationTempdir)
system('mkdir --parents '+ self.integrationTempdir)
self.suggestionServerPort = PortNumberGenerator.next()
self.luceneServerPort = PortNumberGenerator.next()
self.httpPort = PortNumberGenerator.next()
self.testdataDir = join(dirname(mydir), "data")
def testBindPortNumberGiven(self):
port, close = attemptBinding(bindPort=0); close()
PortNumberGenerator.bind(port=port)
self.assertNotBound(bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
if has_dual_stack():
self.assertNotBound(bindV6(ip='::1', port=port, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=port)
self.assertBoundAndUnbind(bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
if has_dual_stack():
self.assertBoundAndUnbind(bindV6(ip='::1', port=port, protocol='tcp', reuse=True))
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self,
"examples-" + stateName,
tests=tests,
fastMode=fastMode)
self.testdataDir = join(dirname(mydir), 'data')
self.gatewayPort = PortNumberGenerator.next()
self.indexPort = PortNumberGenerator.next()
self.apiPort = PortNumberGenerator.next()
self.lucenePort = PortNumberGenerator.next()
self.sruslavePort = PortNumberGenerator.next()
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self, stateName=stateName, tests=tests, fastMode=fastMode)
self.virtuosoDataDir = join(self.integrationTempdir, 'virtuoso-data')
self.bulkLoadDir = join(self.integrationTempdir, 'bulk-load-data')
self.virtuosoPort = PortNumberGenerator.next()
self.odbcPort = PortNumberGenerator.next()
self.httpPort = PortNumberGenerator.next()
self.testdataDir = join(dirname(myDir), 'data')
if not fastMode:
system('rm -rf ' + self.integrationTempdir)
system('mkdir --parents ' + self.virtuosoDataDir)
def __init__(self, stateName, tests=None, fastMode=False):
SeecrIntegrationState.__init__(self, "meresco-lucene-" + stateName, tests=tests, fastMode=fastMode)
self.stateName = stateName
if not fastMode:
system('rm -rf ' + self.integrationTempdir)
system('mkdir --parents '+ self.integrationTempdir)
self.suggestionServerPort = PortNumberGenerator.next()
self.luceneServerPort = PortNumberGenerator.next()
self.httpPort = PortNumberGenerator.next()
self.testdataDir = join(dirname(mydir), "data")
self.JAVA_BIN = "/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/bin"
if not isdir(self.JAVA_BIN):
self.JAVA_BIN = "/usr/lib/jvm/java-1.8.0/bin"
def testBindPortNumbersGeneratedV4(self):
p = PortNumberGenerator.next(bind=True)
self.assertTrue(0 < p < 65536)
for reuse in [True, False]:
self.assertNotBound(
bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=reuse))
self.assertNotBound(
bindV4(ip='127.0.0.1', port=p, protocol='udp', reuse=reuse))
self.assertNotBound(
bindV4(ip='0.0.0.0', port=p, protocol='tcp', reuse=reuse))
self.assertNotBound(
bindV4(ip='0.0.0.0', port=p, protocol='udp', reuse=reuse))
PortNumberGenerator.unbind(p)
def testBindPortRangeNumberGiven(self):
port = PortNumberGenerator.next(blockSize=2)
port2 = port + 1
PortNumberGenerator.bind(port=port, blockSize=2)
self.assertNotBound(
bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
self.assertNotBound(
bindV4(ip='127.0.0.1', port=port2, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=port, blockSize=2)
self.assertBoundAndUnbind(
bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
self.assertBoundAndUnbind(
bindV4(ip='127.0.0.1', port=port2, protocol='tcp', reuse=True))
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self, "harvester-"+stateName, tests=tests, fastMode=fastMode)
self.helperServerPortNumber = PortNumberGenerator.next()
self.harvesterInternalServerPortNumber = PortNumberGenerator.next()
self.helperDir = join(self.integrationTempdir, 'helper')
self.dumpDir = join(self.helperDir, 'dump')
self.harvesterLogDir = join(self.integrationTempdir, "log")
self.harvesterStateDir = join(self.integrationTempdir, "state")
copytree(join("integration-data", "data"), join(self.integrationTempdir, 'data'))
for f in listdir(join(self.integrationTempdir, "data")):
filepath = join(self.integrationTempdir, "data", f)
if isfile(filepath):
fileSubstVars(filepath, helperServerPortNumber=self.helperServerPortNumber, integrationTempdir=self.integrationTempdir)
def testReasonableAmountOfUniquePortNumbers(self):
number = PortNumberGenerator.next()
self.assertEquals(int, type(number))
numbers = []
# More than 14000 gets *very* slow or fails
# When guaranteed uniqe numbers for that many ports are needed,
# change the approach (say reading: cat /proc/net/tcp | awk '{print $2}' | sed -e '1d')
for i in xrange(14000):
numbers.append(PortNumberGenerator.next())
self.assertEquals(14000, len(numbers))
self.assertEquals(14000, len(set(numbers)))
self.assertEquals(True, all((0 <= n < 65536) for n in numbers))
def testReasonableAmountOfUniquePortNumbers(self):
number = PortNumberGenerator.next()
self.assertEqual(int, type(number))
numbers = []
# More than 14000 gets *very* slow or fails
# When guaranteed uniqe numbers for that many ports are needed,
# change the approach (say reading: cat /proc/net/tcp | awk '{print $2}' | sed -e '1d')
for i in range(4000):
numbers.append(PortNumberGenerator.next())
self.assertEqual(4000, len(numbers))
self.assertEqual(4000, len(set(numbers)))
self.assertEqual(True, all((0 <= n < 65536) for n in numbers))
def testHttpAndHttpsGetStreaming(self):
for useSsl in [False, True]:
get_request = []
port = PortNumberGenerator.next()
streamingData = StreamingData(data=[c for c in "STREAMING GET RESPONSE"])
self.referenceHttpServer(port, get_request, ssl=useSsl, streamingData=streamingData)
dataHandled = []
def handleDataFragment(data):
dataHandled.append(data)
if '\r\n\r\n' in ''.join(dataHandled):
streamingData.doNext()
responses = []
def gethandler(*args, **kwargs):
f = httpsget if useSsl else httpget
response = 'no response yet'
try:
response = yield f('localhost', port, '/path',
headers={'Accept': 'text/plain'},
handlePartialResponse=handleDataFragment,
)
finally:
responses.append(response)
self.handler = gethandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertEquals([None], responses)
self.assertTrue("STREAMING GET RESPONSE" in ''.join(dataHandled), dataHandled)
self.assertTrue(len(dataHandled) > len("STREAMING GET RESPONSE"), dataHandled)
self.assertEquals('GET', get_request[0]['command'])
self.assertEquals('/path', get_request[0]['path'])
headers = get_request[0]['headers'].headers
self.assertEquals(['Accept: text/plain\r\n'], headers)
def testSuspendProtocolWithThrow(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
try:
suspend.getResult()
self.fail()
except ValueError, e:
tbstring = format_exc()
yield "result = %s" % tbstring
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
def raiser():
raise ValueError("BAD VALUE")
try:
raiser()
except ValueError, e:
exc_type, exc_value, exc_traceback = exc_info()
suspend.throw(exc_type, exc_value, exc_traceback)
def testConnect(self):
self.req = False
serverResponse = 'Hello World'
def onRequest(**kwargs):
yield 'HTTP/1.0 200 OK\r\n\r\n' + serverResponse
self.req = True
port = PortNumberGenerator.next()
with Reactor() as reactor:
try:
server = HttpsServer(reactor, port, onRequest, keyfile='ssl/server.pkey', certfile='ssl/server.cert')
server.listen()
p = Popen('wget -O - --no-check-certificate --quiet https://localhost:%s' % port, shell=True, stdout=PIPE)
popenStdout = []
def readPopenStdout():
popenStdout.append(p.stdout.read())
reactor.addReader(p.stdout, readPopenStdout)
while not self.req:
reactor.step()
reactor.step()
self.assertEquals(1, len(popenStdout))
self.assertEquals(serverResponse, popenStdout[0])
finally:
server.shutdown()
# cleanup
reactor.removeReader(p.stdout)
def testServerBindAddress(self):
reactor = CallTrace()
port = PortNumberGenerator.next()
server = ObservableHttpServer(reactor, port, bindAddress='127.0.0.1')
server.startServer()
self.assertEquals(('127.0.0.1', port),
server._httpserver._acceptor._sok.getsockname())
def testHttpsGet(self):
get_request = []
port = PortNumberGenerator.next()
self.referenceHttpServer(port, get_request, ssl=True)
responses = []
def gethandler(*args, **kwargs):
response = yield httpsget('localhost',
port,
'/path',
headers={
'Content-Type': 'text/plain',
'Content-Length': 0
})
yield response
responses.append(response)
self.handler = gethandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertTrue("GET RESPONSE" in responses[0], responses[0])
self.assertEquals('GET', get_request[0]['command'])
self.assertEquals('/path', get_request[0]['path'])
headers = get_request[0]['headers'].headers
self.assertEquals(
['Content-Length: 0\r\n', 'Content-Type: text/plain\r\n'], headers)
def testHttpRequest(self):
get_request = []
port = PortNumberGenerator.next()
self.referenceHttpServer(port, get_request)
responses = []
def gethandler(*args, **kwargs):
response = 'no response yet'
try:
response = yield httprequest(method='MYMETHOD',
host='localhost',
port=port,
request='/path',
headers={
'Content-Type': 'text/plain',
'Content-Length': 0
},
prio=4)
finally:
responses.append(response)
yield 'HTTP/1.0 200 OK\r\n\r\n'
self.handler = gethandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertTrue(
"Message: Unsupported method ('MYMETHOD')" in responses[0],
responses[0])
def setUp(self):
TestCase.setUp(self)
self.tempdir = mkdtemp()
fd, self.tempfile = mkstemp()
os.close(fd)
self.reactor = Reactor()
self.port = PortNumberGenerator.next()
def testHttpsPost(self):
post_request = []
port = PortNumberGenerator.next()
self.referenceHttpServer(port, post_request, ssl=True)
body = u"BÖDY" * 20000
responses = []
def posthandler(*args, **kwargs):
response = yield httpspost('localhost',
port,
'/path',
body,
headers={'Content-Type': 'text/plain'})
yield response
responses.append(response)
self.handler = posthandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertTrue("POST RESPONSE" in responses[0], responses[0])
self.assertEquals('POST', post_request[0]['command'])
self.assertEquals('/path', post_request[0]['path'])
headers = post_request[0]['headers'].headers
self.assertEquals(
['Content-Length: 100000\r\n', 'Content-Type: text/plain\r\n'],
headers)
self.assertEquals(body, post_request[0]['body'])
def testCloseStatesLocalFirst(self):
remoteport = PortNumberGenerator.next()
server = socket()
server.bind(('localhost', remoteport))
server.listen(1)
local = socket()
local.connect(('localhost', remoteport))
localport = local.getsockname()[1]
remote = server.accept()[0]
self.assertNetStat(remoteport, localport, 'ESTABLISHED')
self.assertNetStat(localport, remoteport, 'ESTABLISHED')
local.close()
self.assertNetStat(localport, remoteport, 'FIN_WAIT')
self.assertNetStat(remoteport, localport, 'CLOSE_WAIT')
remote.close()
self.assertNetStat(localport, remoteport, 'TIME_WAIT')
self.assertNetStat(remoteport, localport, None)
server = socket()
try:
server.bind(('127.0.0.1', remoteport))
except:
self.fail('bind must succeed')
def testBindPortNumberGiven(self):
port, close = attemptBinding(bindPort=0)
close()
PortNumberGenerator.bind(port=port)
self.assertNotBound(
bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
if has_dual_stack():
self.assertNotBound(
bindV6(ip='::1', port=port, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=port)
self.assertBoundAndUnbind(
bindV4(ip='127.0.0.1', port=port, protocol='tcp', reuse=True))
if has_dual_stack():
self.assertBoundAndUnbind(
bindV6(ip='::1', port=port, protocol='tcp', reuse=True))
def testSuspendProtocol(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
yield "result = %s" % suspend.getResult()
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
self.assertEquals(1, len(reactor._fds))
self.assertEquals([READ_INTENT], [v.intent for v in reactor._fds.values()])
reactor.step()
self.assertEquals(1, len(reactor._fds))
self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()])
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
suspend.resume('RESPONSE')
self.assertEquals(1, len(reactor._fds))
self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()])
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(['before suspend', 'result = RESPONSE', 'after suspend'], listener.data)
self.assertEquals(0, len(reactor._fds))
# cleanup (most) fd's
listener.close()
def testBindPortNumbersGeneratedV6(self):
if not has_dual_stack():
return printDualStackSkipped()
p = PortNumberGenerator.next(bind=True)
self.assertTrue(0 < p < 65536)
for protocol in ['tcp', 'udp']:
for reuse in [True, False]:
self.assertNotBound(
bindV4(ip='127.0.0.1',
port=p,
protocol=protocol,
reuse=reuse))
self.assertNotBound(
bindV4(ip='0.0.0.0',
port=p,
protocol=protocol,
reuse=reuse))
self.assertNotBound(
bindV6(ip='::1', port=p, protocol=protocol, reuse=reuse))
self.assertNotBound(
bindV6(ip='::', port=p, protocol=protocol, reuse=reuse))
self.assertNotBound(
bindV6(ip='::1',
port=p,
protocol=protocol,
reuse=reuse,
ipV6Only=False))
self.assertNotBound(
bindV6(ip='::',
port=p,
protocol=protocol,
reuse=reuse,
ipV6Only=False))
def setUp(self):
TestCase.setUp(self)
self.tempdir = mkdtemp()
fd, self.tempfile = mkstemp()
os.close(fd)
self.reactor = Reactor()
self.port = PortNumberGenerator.next()
def testSuspendProtocolWithThrow(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
try:
suspend.getResult()
self.fail()
except ValueError, e:
tbstring = format_exc()
yield "result = %s" % tbstring
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
def raiser():
raise ValueError("BAD VALUE")
try:
raiser()
except ValueError, e:
exc_type, exc_value, exc_traceback = exc_info()
suspend.throw(exc_type, exc_value, exc_traceback)
def __init__(self, stateName, tests=None, fastMode=False):
SeecrIntegrationState.__init__(self,
"meresco-lucene-" + stateName,
tests=tests,
fastMode=fastMode)
self.stateName = stateName
if not fastMode:
system('rm -rf ' + self.integrationTempdir)
system('mkdir --parents ' + self.integrationTempdir)
self.suggestionServerPort = PortNumberGenerator.next()
self.luceneServerPort = PortNumberGenerator.next()
self.httpPort = PortNumberGenerator.next()
self.testdataDir = join(dirname(mydir), "data")
self.JAVA_BIN = "/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/bin"
if not isdir(self.JAVA_BIN):
self.JAVA_BIN = "/usr/lib/jvm/java-1.8.0/bin"
def testBindPortNumbersGenerated_withBlockSize_V4(self):
blockSize = 3
p = PortNumberGenerator.next(bind=True, blockSize=blockSize)
for consequative_p in range(p, p + blockSize):
for reuse in [False, True]:
self.assertNotBound(
bindV4(ip='127.0.0.1',
port=consequative_p,
protocol='tcp',
reuse=reuse))
self.assertNotBound(
bindV4(ip='127.0.0.1',
port=consequative_p,
protocol='udp',
reuse=reuse))
self.assertNotBound(
bindV4(ip='0.0.0.0',
port=consequative_p,
protocol='tcp',
reuse=reuse))
self.assertNotBound(
bindV4(ip='0.0.0.0',
port=consequative_p,
protocol='udp',
reuse=reuse))
def testHttpRequestWithTimeout(self):
# And thus too http(s)get/post/... and friends.
get_request = []
port = PortNumberGenerator.next()
def slowData():
for i in xrange(5):
yield i
sleep(0.01)
responses = []
def handlerFactory(timeout):
def gethandler(*args, **kwargs):
try:
response = yield httprequest(
method='GET',
host='localhost',
port=port,
request='/path',
headers={
'Content-Type': 'text/plain',
'Content-Length': 0,
'Host': 'localhost'
},
timeout=timeout,
)
responses.append(response)
except TimeoutException, e:
responses.append(e)
finally:
assert responses, 'Either a timeout or response should have occurred.'
def __init__(self, stateName, tests=None, fastMode=False):
_IntegrationState.__init__(self,
"meresco-html-server-" + stateName,
tests=tests,
fastMode=fastMode)
self.port = PortNumberGenerator.next()
def testCloseStatesRemoteFirst(self):
remoteport = PortNumberGenerator.next()
server = socket()
server.bind(('127.0.0.1', remoteport))
server.listen(1)
local = socket()
local.connect(('127.0.0.1', remoteport))
localport = local.getsockname()[1]
remote = server.accept()[0]
self.assertNetStat(remoteport, localport, 'ESTABLISHED')
self.assertNetStat(localport, remoteport, 'ESTABLISHED')
remote.close()
self.assertNetStat(remoteport, localport, 'FIN_WAIT')
self.assertNetStat(localport, remoteport, 'CLOSE_WAIT')
local.close()
self.assertNetStat(remoteport, localport, 'TIME_WAIT')
self.assertNetStat(localport, remoteport, None)
server.close()
server = socket()
try:
server.bind(('127.0.0.1', remoteport))
self.fail('re-bind must raise Address Already in Use Exception')
except AssertionError:
raise
except Exception, e:
pass
def testHttpGet(self):
get_request = []
port = PortNumberGenerator.next()
self.referenceHttpServer(port, get_request)
responses = []
def gethandler(*args, **kwargs):
response = 'no response yet'
try:
response = yield httpget('localhost', port, '/path',
headers={'Content-Type': 'text/plain', 'Content-Length': 0},
prio=4
)
finally:
responses.append(response)
yield 'HTTP/1.0 200 OK\r\n\r\n'
self.handler = gethandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
self.assertTrue("GET RESPONSE" in responses[0], responses[0])
self.assertEquals('GET', get_request[0]['command'])
self.assertEquals('/path', get_request[0]['path'])
headers = get_request[0]['headers'].headers
self.assertEquals(['Content-Length: 0\r\n', 'Content-Type: text/plain\r\n'], headers)
def testHttpGetWithReallyLargeHeaders(self):
get_request = []
port = PortNumberGenerator.next()
self.referenceHttpServer(port, get_request)
responses = []
headersOrig = {'Accept': 'text/plain'}
headersOrig.update([
('X-Really-Largely-Large-%s' % i, 'aLargelyLargeValue')
for i in range(10000)
])
def gethandler(*args, **kwargs):
response = 'no response yet'
try:
response = yield httpget('localhost', port, '/path',
headers=headersOrig,
)
finally:
responses.append(response)
yield 'HTTP/1.0 200 OK\r\n\r\n'
self.handler = gethandler
clientget('localhost', self.port, '/')
self._loopReactorUntilDone()
headers = get_request[0]['headers'].headers
headersAsDict = dict([tuple(h.strip().split(': ', 1)) for h in headers])
self.assertEquals(len(headersOrig), len(headersAsDict))
self.assertEquals(headersOrig, headersAsDict)
self.assertTrue("GET RESPONSE" in responses[0], responses[0])
self.assertEquals('GET', get_request[0]['command'])
self.assertEquals('/path', get_request[0]['path'])
def testUnbindPortNumberV4(self):
p = PortNumberGenerator.next(bind=True)
self.assertNotBound(
bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=p)
self.assertBoundAndUnbind(
bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=False))
self.assertBoundAndUnbind(
bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=False)
) # Same IP-version, protocol, host, port combination: basically testing bindV4's close happened.
self.assertBoundAndUnbind(
bindV4(ip='127.0.0.1', port=p, protocol='udp', reuse=False))
self.assertBoundAndUnbind(
bindV4(ip='0.0.0.0', port=p, protocol='tcp', reuse=False))
self.assertBoundAndUnbind(
bindV4(ip='0.0.0.0', port=p, protocol='udp', reuse=False))
def posthandler(*args, **kwargs):
response = yield httpspost('localhost',
PortNumberGenerator.next(),
'/path',
"body",
headers={'Content-Type': 'text/plain'})
yield response
responses.append(response)
def testBindPortNumbersGeneratedV4(self):
p = PortNumberGenerator.next(bind=True)
self.assertTrue(0 < p < 65536)
for reuse in [True, False]:
self.assertNotBound(bindV4(ip='127.0.0.1', port=p, protocol='tcp', reuse=reuse))
self.assertNotBound(bindV4(ip='127.0.0.1', port=p, protocol='udp', reuse=reuse))
self.assertNotBound(bindV4(ip='0.0.0.0', port=p, protocol='tcp', reuse=reuse))
self.assertNotBound(bindV4(ip='0.0.0.0', port=p, protocol='udp', reuse=reuse))
def testUnbindPortNumberV6(self):
if not has_dual_stack():
return printDualStackSkipped()
p = PortNumberGenerator.next(bind=True)
self.assertNotBound(bindV4(ip='::1', port=p, protocol='tcp', reuse=True))
PortNumberGenerator.unbind(port=p)
self.assertBoundAndUnbind(bindV6(ip='::1', port=p, protocol='tcp', reuse=False)) # Same IP-version, protocol, host, port combination: basically testing bindV6's close happened.
for protocol in ['tcp', 'udp']:
self.assertBoundAndUnbind(bindV4(ip='127.0.0.1', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(bindV4(ip='0.0.0.0', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(bindV6(ip='::1', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(bindV6(ip='::', port=p, protocol=protocol, reuse=False))
self.assertBoundAndUnbind(bindV6(ip='::1', port=p, protocol=protocol, reuse=False, ipV6Only=False))
self.assertBoundAndUnbind(bindV6(ip='::', port=p, protocol=protocol, reuse=False, ipV6Only=False))
def testSuspendThrowBackwardsCompatibleWithInstanceOnlyThrow_YouWillMissTracebackHistory(
self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
try:
suspend.getResult()
self.fail()
except ValueError, e:
tbstring = format_exc()
yield "result = %s" % tbstring
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
self.assertEquals(1, len(reactor._fds))
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
def raiser():
raise ValueError("BAD VALUE")
try:
raiser()
except:
exc_value = exc_info()[1]
suspend.throw(exc_value)
self.assertEquals(1, len(reactor._fds))
reactor.step()
reactor.step()
reactor.step()
expectedTraceback = ignoreLineNumbers(
"""Traceback (most recent call last):
File "%(__file__)s", line 201, in handler
suspend.getResult()
File "%(suspend.py)s", line 62, in getResult
raise self._exception[0], self._exception[1], self._exception[2]
ValueError: BAD VALUE
""" % fileDict)
self.assertEquals(3, len(listener.data))
self.assertEquals('before suspend', listener.data[0])
self.assertEqualsWS("result = %s" % expectedTraceback,
ignoreLineNumbers(listener.data[1]))
self.assertEquals('after suspend', listener.data[2])
self.assertEquals(0, len(reactor._fds))
# cleanup (most) fd's
listener.close()
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self, stateName, tests=tests, fastMode=fastMode)
self.testdataDir = join(mydir, 'data')
self.digitaleCollectiePort = PortNumberGenerator.next()
self.erfGeoEnrichmentPort = PortNumberGenerator.next()
self.erfGeoEnrichmentLocalStatePath = join(self.integrationTempdir, 'erfGeoEnrichmentLocal')
self.erfGeoEnrichmentIndexPort = PortNumberGenerator.next()
self.erfGeoEnrichmentIndexLocalStatePath = join(self.integrationTempdir, 'erfGeoEnrichmentIndexLocal')
erfGeoRepositorySetsSelectionFile = join(self.erfGeoEnrichmentLocalStatePath, 'erfgeo_dc_sets.json')
if not self.fastMode:
clearOrCreateDir(self.erfGeoEnrichmentLocalStatePath)
open(erfGeoRepositorySetsSelectionFile, 'w').write(ERFGEO_SETS_SELECTION_JSON)
self.erfGeoApiPort = PortNumberGenerator.next()
self.globalStatePath = join(self.integrationTempdir, 'global')
self.saharaGetPort = PortNumberGenerator.next()
self.config = config = readConfig(join(documentationDir, 'examples', 'dc-erfgeo-enrich.config'))
# test example config has necessary parameters
def setConfig(config, parameter, value):
assert config.get(parameter), "key '%s' should only be given a value if it is already declared in source config %s." % (parameter, config)
print "config[%s] = %s" % (repr(parameter), repr(value))
config[parameter] = value
setConfig(config, 'erfgeoEnrich.portNumber', self.erfGeoEnrichmentPort)
setConfig(config, 'erfgeoEnrich.index.portNumber', self.erfGeoEnrichmentIndexPort)
setConfig(config, 'digitaleCollectie.host', 'localhost')
setConfig(config, 'digitaleCollectie.port', self.digitaleCollectiePort)
setConfig(config, 'erfgeo.searchApiBaseUrl', 'http://localhost:%s' % self.erfGeoApiPort)
config['global.apacheLogStream'] = 'disabled'
config['global.debug.periodicdownload.period'] = '0.1'
self.configFile = join(self.integrationTempdir, 'erfgeo.config')
with open(self.configFile, 'w') as f:
for item in config.items():
f.write('%s = %s\n' % item)
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self, stateName=stateName, tests=tests, fastMode=fastMode)
self.jenaDataDir = join(self.integrationTempdir, 'jena-data')
self.jenaPort = PortNumberGenerator.next()
self.testdataDir = join(dirname(myDir), 'data')
if not fastMode:
system('rm -rf ' + self.integrationTempdir)
system('mkdir --parents ' + self.jenaDataDir)
def testClaimBlockOfPortNumbers(self):
ports = []
for i in xrange(20):
p = PortNumberGenerator.next(blockSize=3)
self.assertTrue(p in PortNumberGenerator._usedPorts)
self.assertTrue((p + 1) in PortNumberGenerator._usedPorts)
self.assertTrue((p + 2) in PortNumberGenerator._usedPorts)
ports.append(p)
self.assertEquals(20, len(ports))
def testListenAndLog(self):
port = PortNumberGenerator.next()
ulal = UdpListenAndLog(port)
s = socket(AF_INET, SOCK_DGRAM)
s.sendto("THIS IS THE PAYLOAD", ('127.0.0.1', port))
sleep(.5)
self.assertEquals(['THIS IS THE PAYLOAD'], ulal.log())
def testClaimBlockOfPortNumbers(self):
ports = []
for i in range(20):
p = PortNumberGenerator.next(blockSize=3)
self.assertTrue(p in PortNumberGenerator._usedPorts)
self.assertTrue((p + 1) in PortNumberGenerator._usedPorts)
self.assertTrue((p + 2) in PortNumberGenerator._usedPorts)
ports.append(p)
self.assertEqual(20, len(ports))
def __init__(self, stateName, tests=None, fastMode=False):
IntegrationState.__init__(self, stateName, tests=tests, fastMode=fastMode)
self.testdataDir = join(dirname(mydir), 'data/integration')
stateDataDir = join(dirname(mydir), 'data', stateName)
if isdir(stateDataDir):
self.testdataDir = stateDataDir
self.solrDataDir = join(self.integrationTempdir, "solr")
self.owlimDataDir = join(self.integrationTempdir, "owlim")
self.httpDataDir = join(mydir, "httpdata")
system('mkdir --parents ' + self.solrDataDir)
if not self.fastMode:
system('cp -r %s/* %s' % (defaultSolrDataDir, self.solrDataDir))
system("sed 's,^jetty\.home=.*$,jetty.home=%s,' -i %s" % (self.solrDataDir, join(self.solrDataDir, 'start.config')))
self.solrPortNumber = PortNumberGenerator.next()
self.owlimPortNumber = PortNumberGenerator.next()
self.portNumber = PortNumberGenerator.next()
self.httpPortNumber = PortNumberGenerator.next()
self.hostName = 'localhost'
self.config = config = readConfig(join(documentationDir, 'examples', 'oas.config'))
self.configFile = join(self.integrationTempdir, 'oas.config')
# test example config has necessary parameters
def setConfig(config, parameter, value):
assert config.get(parameter)
config[parameter] = value
setConfig(config, 'hostName', self.hostName)
setConfig(config, 'portNumber', self.portNumber)
setConfig(config, 'solrPortNumber', self.solrPortNumber)
setConfig(config, 'owlimPortNumber', self.owlimPortNumber)
databasePath = join(self.integrationTempdir, 'database')
self.harvesterDataDir = join(databasePath, 'harvester')
setConfig(config, 'databasePath', databasePath)
setConfig(config, 'resolveBaseUrl', 'http://localhost:%s/resolve/' % self.portNumber)
self.publicDocumentationPath = join(documentationDir, 'public')
setConfig(config, 'publicDocumentationPath', self.publicDocumentationPath)
with open(self.configFile, 'w') as f:
for item in config.items():
f.write('%s = %s\n' % item)
def testListenAndLog(self):
port = PortNumberGenerator.next()
ulal = UdpListenAndLog(port)
with socket(AF_INET, SOCK_DGRAM) as s:
s.sendto(b"THIS IS THE PAYLOAD", ('127.0.0.1', port))
sleep(.5)
self.assertEqual([b'THIS IS THE PAYLOAD'], ulal.log())
ulal.stop()
def testBindPortNumbersGenerated_withBlockSize_V4(self):
blockSize = 3
p = PortNumberGenerator.next(bind=True, blockSize=blockSize)
for consequative_p in range(p, p + blockSize):
for reuse in [False, True]:
self.assertNotBound(bindV4(ip='127.0.0.1', port=consequative_p, protocol='tcp', reuse=reuse))
self.assertNotBound(bindV4(ip='127.0.0.1', port=consequative_p, protocol='udp', reuse=reuse))
self.assertNotBound(bindV4(ip='0.0.0.0', port=consequative_p, protocol='tcp', reuse=reuse))
self.assertNotBound(bindV4(ip='0.0.0.0', port=consequative_p, protocol='udp', reuse=reuse))
def __init__(self, stateName, tests=None, fastMode=False):
SeecrIntegrationState.__init__(self,
"meresco-uri-enumerate-" + stateName,
tests=tests,
fastMode=fastMode)
self.stateName = stateName
if not fastMode:
system('rm -rf ' + self.integrationTempdir)
system('mkdir --parents ' + self.integrationTempdir)
self.numerateServerPort = PortNumberGenerator.next()
self.testdataDir = join(dirname(mydir), "data")
def testHangupConnectionTimeout(self):
expectedException = IOError if PY_VERSION == "2.7" else URLError
self.ms = MockServer(port=PortNumberGenerator.next(), hangupConnectionTimeout=0.1)
self.ms.start()
t0 = time()
self.assertRaises(expectedException, lambda: urlopen(self.ms.myUrl).read())
t1 = time()
delta = t1 - t0
self.assertTrue(0.09 < delta < 0.12, "Expected around 0.1, was %s" % delta)
self.assertEquals(0, len(self.ms.requests))
def testStartListening(self):
reactor = CallTrace()
port = PortNumberGenerator.next()
Acceptor(reactor, port, lambda sok: lambda: None)
self.assertEquals('addReader', reactor.calledMethods[0].name)
sok = reactor.calledMethods[0].args[0]
out = Popen(['netstat', '--numeric', '--listening', '--udp'], stdout=PIPE, stderr=PIPE).communicate()[0]
self.assertTrue(str(port) in out, out)
sok.close()
callback = reactor.calledMethods[0].args[1]
self.assertTrue(callable(callback))
def testHangupConnectionTimeout(self):
expectedException = IOError
self.ms.socket.close()
self.ms = MockServer(port=PortNumberGenerator.next(), hangupConnectionTimeout=0.1)
self.ms.start()
t0 = time()
self.assertRaises(expectedException, lambda: urlopen(self.ms.myUrl).read())
t1 = time()
delta = t1 - t0
self.assertTrue(0.09 < delta < 0.12, "Expected around 0.1, was %s" % delta)
self.assertEqual(0, len(self.ms.requests))
