def testChunkedAllTheWay(self):
reactor = CallTrace('Reactor')
sokket = CallTrace('Sokket')
data = []
class Handler:
def send(self, chunk):
data.append(chunk)
def throw(self, exception):
pass
httpreader = HttpReader(reactor, sokket, Handler(), 'GET', 'host,nl',
'/')
httpreader._chunked = True
# chunk == network message
httpreader._sendFragment('4\r\n1234\r\n')
self.assertEquals(['1234'], data)
httpreader._sendFragment('10\r\n0123456789abcdef\r\n')
self.assertEquals(['1234', '0123456789abcdef'], data)
del data[0]
del data[0]
# chunk = 2 network messages
httpreader._sendFragment('10\r\nfedcba')
#self.assertEquals(['fedcba'], data)
httpreader._sendFragment('9876543210\r\n')
self.assertEquals(['fedcba9876543210'], data)
def testExceptionInDownstreamHandlePacket(self):
reactor = CallTrace()
server = UdpPacketListener(reactor=reactor, port=1234)
try:
sok = CallTrace(
returnValues={'recvfrom': (b'data', ('127.0.0.1', 1234))})
observer = CallTrace()
observer.exceptions['handlePacket'] = Exception(
"This should be happening")
server.addObserver(observer)
mockStderr = StringIO()
sys.stderr = mockStderr
try:
server._handlePacket(sok)
reactor.calledMethods[-1].args[0]()
finally:
sys.stderr = sys.__stderr__
lines = mockStderr.getvalue().split('\n')
self.assertEqual(
"Exception in _handlePacket for data='data' from ('127.0.0.1', 1234)",
lines[0])
self.assertEqual('Traceback (most recent call last):', lines[1])
self.assertEqual('Exception: This should be happening', lines[-2])
finally:
server.close()
def setUp(self):
SeecrTestCase.setUp(self)
self.observer1 = CallTrace('observer1',
emptyGeneratorMethods=['message'],
returnValues={
'function': 41,
'gen': (i for i in [41]),
'noop': None
})
self.observer2 = object()
self.observer3 = CallTrace('observer3',
emptyGeneratorMethods=['message'],
returnValues={
'function': 42,
'gen': (i for i in [42]),
'noop': None
})
self.dna = be(
(Observable(), (FilterMessages(
disallowed=['message', 'function', 'gen', 'noop']),
(self.observer1, )),
(FilterMessages(allowed=['message', 'function', 'gen', 'noop']),
(self.observer2, )),
(FilterMessages(allowed=['message', 'function', 'gen', 'noop']),
(self.observer3, ))))
def testCouldHaveButDoesnot(self):
inputEvent = fromstring('<document><other/></document>')
interceptor = CallTrace('Interceptor',
ignoredAttributes=[
'getData', 'all_unknown', 'any_unknown',
'call_unknown'
],
methods={'add': yieldNothing})
storage = CallTrace('Storage',
ignoredAttributes=['add', 'all_unknown'])
storage.exceptions['getData'] = KeyError('Part not available')
v = createVenturiHelix([{
'partname': 'other',
'xpath': '/document/other'
}], [{
'partname': 'one',
'xpath': '/document/one'
}], interceptor, storage)
list(compose(v.all.add('identifier', 'document', inputEvent)))
self.assertEquals(['begin', 'add'],
[m.name for m in interceptor.calledMethods])
self.assertEquals('identifier',
interceptor.calledMethods[1].kwargs['identifier'])
self.assertEquals('other',
interceptor.calledMethods[1].kwargs['partname'])
def testShouldAbortInitWithExceptionIfRunningMarkerStillThere(self):
# Subclassed SystemExit for ShutdownFailedException to ensure a service quits when left in an inconsistent state,
# even if it needs to traverse "except (SystemExit, KeyboardInterrupt, AssertionError)" catch-statements.
self.assertTrue(issubclass(ShutdownFailedException, SystemExit))
with open(join(self.tempdir, 'running.marker'), 'w') as fp:
fp.write('already there')
with stdout_replaced() as output:
try:
self._createShutdownHandler(statePath=self.tempdir,
server=CallTrace('server'),
reactor=CallTrace('reactor'))
self.fail('should terminate')
except ShutdownFailedException:
pass
self.assertTrue(
"process was not previously shutdown to a consistent persistent state."
in output.getvalue(), output.getvalue())
self.assertTrue(
"NOT starting from an unknown state." in output.getvalue(),
output.getvalue())
with stdout_replaced() as output:
try:
self._createShutdownHandler(statePath=self.tempdir,
server=CallTrace('server'),
reactor=CallTrace('reactor'))
self.fail('should terminate')
except SystemExit:
pass
def testHandleRequest(self):
observer = CallTrace('Observer',
methods={
'handleRequest': lambda *a, **kw:
(x for x in [])
})
s = ObservableHttpServer(CallTrace('Reactor'), 1024)
s.addObserver(observer)
list(
compose(
s.handleRequest(
RequestURI=
'http://localhost/path?key=value&emptykey#fragment')))
self.assertEquals(1, len(observer.calledMethods))
method = observer.calledMethods[0]
self.assertEquals('handleRequest', method.name)
self.assertEquals(0, len(method.args))
self.assertEquals(7, len(method.kwargs))
self.assertTrue('arguments' in method.kwargs, method.kwargs)
arguments = method.kwargs['arguments']
self.assertEquals(2, len(arguments))
self.assertEquals(['emptykey', 'key'], sorted(arguments.keys()))
self.assertEquals(['value'], arguments['key'])
self.assertEquals([''], arguments['emptykey'])
def testAddDomain(self):
header, body = parseResponse(
asBytes(
self.dna.all.handleRequest(
user=CallTrace(returnValues=dict(isAdmin=False)),
path="/actions/addDomain",
Body=bytes(urlencode(dict(identifier="aap")),
encoding="utf-8"),
Method='Post')))
self.assertEqual(0, len(self.observable.calledMethods))
self.assertEqual("200", header['StatusCode'])
self.assertEqual("application/json", header['Headers']['Content-Type'])
response = JsonDict.loads(body)
self.assertFalse(response['success'])
self.assertEqual("Not allowed", response['message'])
header, body = parseResponse(
asBytes(
self.dna.all.handleRequest(
user=CallTrace(returnValues=dict(isAdmin=True)),
path="/actions/addDomain",
Body=bytes(urlencode(dict(identifier="aap")),
encoding="utf-8"),
Method='Post')))
self.assertEqual("200", header['StatusCode'])
self.assertEqual("application/json", header['Headers']['Content-Type'])
response = JsonDict.loads(body)
self.assertTrue(response['success'])
self.assertEqual(1, len(self.observable.calledMethods))
self.assertEqual("addDomain", self.observable.calledMethods[0].name)
self.assertEqual(dict(identifier='aap'),
self.observable.calledMethods[0].kwargs)
def testLastRecvContainsCompleteChunk(self):
reactor = CallTrace('Reactor')
sokket = CallTrace('Sokket')
data = []
done = []
class Handler:
def send(self, chunk):
data.append(chunk)
def throw(self, exception):
done.append(True)
httpreader = HttpReader(reactor, sokket, Handler(), 'GET', 'host,nl',
'/')
httpreader._chunked = True
chunkOne = '9\r\n123456789\r\n'
chunkTwo = '8\r\n87654321\r\n'
httpreader._sendFragment(chunkOne)
self.assertEquals(['123456789'], data)
httpreader._sendFragment(chunkTwo)
self.assertEquals(['123456789', '87654321'], data)
while data:
del data[0] # now both in one network message
httpreader._sendFragment(chunkOne + chunkTwo + '0\r\n\r\n')
# Send fragment will only read one fragment.
# Now feed it until all chunks are finished
httpreader._sendFragment('')
httpreader._sendFragment('')
self.assertEquals(['123456789', '87654321'], data)
self.assertEquals([True], done)
def testAnalyseLog(self):
observer = CallTrace()
def handleRequest(**kwargs):
yield "We did it!"
handleRequestMock = CallTrace(methods={'handleRequest': handleRequest})
dna = be(
(Observable(),
(LogCollector(),
(GustosLogWriter(),
(AgentCountReport(gustosGroup="Gustos Group", scopeNames=('mock', )), ),
(observer, )
),
(LogCollectorScope("mock"),
(HandleRequestLog(),
(handleRequestMock, )
)
)
)
)
)
result = list(compose(dna.all.handleRequest(Headers={'User-Agent': GOOGLE_BOT})))
self.assertEqual(b"We did it!", result[0])
valuesKwarg = observer.calledMethods[-1].kwargs['values']
self.assertEqual({'Gustos Group': {'User agents': {'Googlebot/2.1': {'count': 1}}}}, valuesKwarg)
def testOne(self):
observable = Observable()
bitMatrixRow = CallTrace('BitMatrixRow')
multi = MultiLevelDrilldown({'date': [('datelevel1', 10, False)]})
drilldown = CallTrace('Drilldown')
def dd(*args, **kwargs):
return iter([('datelevel1', iter([('2008', 13), ('2007', 10)]))])
yield
drilldown.methods['drilldown'] = dd
multi.addObserver(drilldown)
observable.addObserver(multi)
result = list(
compose(observable.call.multiLevelDrilldown(
bitMatrixRow, ['date'])))
self.assertEqual(1, len(drilldown.calledMethods))
drilldownMethod = drilldown.calledMethods[0]
self.assertEqual('drilldown', drilldownMethod.name)
self.assertEqual((bitMatrixRow, [('datelevel1', 10, False)]),
drilldownMethod.args)
self.assertEqual(1, len(result))
(inputFieldName, realFieldName), termCounts = result[0]
self.assertEqual('date', inputFieldName)
self.assertEqual('datelevel1', realFieldName)
self.assertEqual([('2008', 13), ('2007', 10)], list(termCounts))
def testReadFromStorage(self):
inputEvent = fromstring('<document/>')
interceptor = CallTrace('Interceptor',
ignoredAttributes=[
'getData', 'all_unknown', 'any_unknown',
'call_unknown'
])
interceptor.methods['add'] = yieldNothing
storage = CallTrace('Storage',
ignoredAttributes=['add', 'all_unknown'])
storage.returnValues['getData'] = '<some>this is partone</some>'
v = createVenturiHelix(
[{
'partname': 'partone',
'xpath': '/document/part[@name="partone"]/text()'
}], [], interceptor, storage)
list(compose(v.all.add('identifier', 'document', inputEvent)))
self.assertEquals(['begin', 'add'],
[m.name for m in interceptor.calledMethods])
self.assertEquals(
'<some>this is partone</some>',
lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode']))
self.assertEquals(['begin', 'getData'], storage.calledMethodNames())
self.assertEquals(dict(identifier='identifier', name='partone'),
storage.calledMethods[1].kwargs)
def testSameAddFieldGeneratedTwoTimes(self):
__callstack_var_tx__ = CallTrace('TX')
__callstack_var_tx__.locals = {'id': 'identifier'}
intercept = CallTrace(methods={'add': add})
fields2Xml = Fields2Xml('extra')
fields2Xml.addObserver(intercept)
def f():
f = yield fields2Xml.beginTransaction()
yield f
f = compose(f()).next()
f.addField('key.sub', 'value')
f.addField('key.sub', 'othervalue')
f.addField('key.sub', 'value')
f.addField('key.sub', 'separatedbyvalue')
list(compose(f.commit()))
self.assertEquals(['add'], [m.name for m in intercept.calledMethods])
self.assertEquals(
dict(
identifier='identifier',
partname='extra',
data=
'<extra><key><sub>value</sub></key><key><sub>othervalue</sub></key><key><sub>value</sub></key><key><sub>separatedbyvalue</sub></key></extra>'
), intercept.calledMethods[0].kwargs)
def testShouldUnregister(self):
# ... for testing purposes.
def getRelevantHandlers():
return [getsignal(sh)
for sh in SHUTDOWN_SIGNALS] # SIGTERM, SIGINT
persistable = CallTrace('Observer',
emptyGeneratorMethods=['handleShutdown'])
top = be((
Observable(),
(persistable, ),
))
reactor = CallTrace('reactor')
pre = getRelevantHandlers()
handler = registerShutdownHandler(statePath=self.tempdir,
server=top,
reactor=reactor)
_with = getRelevantHandlers()
handler.unregister()
post = getRelevantHandlers()
# shutdownHandler set
ourHandlerSet = set(_with).difference(set(pre))
self.assertEqual(1, len(ourHandlerSet))
ourHandler = ourHandlerSet.pop()
# shutdownHandler removed on .unregister()
self.assertEqual(pre, post)
# shutdownHandler is who he's supposed to be.
self.assertEqual('_handleShutdown', ourHandler.__func__.__name__)
self.assertEqual(handler, ourHandler.__self__)
def testGetRecordWithRepositoryIdentifier(self):
oaigetrecord = OaiGetRecord(OaiRepository(identifier='example.org'))
record = CallTrace('record')
record.identifier = 'id0'
record.prefixes = ['oai_dc']
record.sets = []
record.isDeleted = False
observer = CallTrace(
returnValues={
'isKnownPrefix': True,
'getRecord': record
},
emptyGeneratorMethods=['oaiWatermark', 'oaiRecord'])
oaigetrecord.addObserver(observer)
consume(
oaigetrecord.getRecord(arguments=dict(
verb=['GetRecord'],
metadataPrefix=['oai_dc'],
identifier=['oai:example.org:id0'],
),
**self.httpkwargs))
self.assertEqual(
['getRecord', 'isKnownPrefix', 'oaiWatermark', 'oaiRecord'],
observer.calledMethodNames())
self.assertEqual(dict(identifier='id0', metadataPrefix='oai_dc'),
observer.calledMethods[0].kwargs)
def testTcpPacketListener(self):
reactor = CallTrace('reactor')
observer = CallTrace('observer',
emptyGeneratorMethods=['handlePacket'])
tcpPacketListener = TcpPacketListener(reactor, port=1234)
server = be((Observable(), (tcpPacketListener, (observer, ))))
list(compose(server.once.observer_init()))
self.assertEquals('addReader', reactor.calledMethods[0].name)
acceptCallback = reactor.calledMethods[0].args[1]
data = "TEST" * 1024
sok = socket()
sok.connect(('localhost', 1234))
bytesSent = sok.send(data)
self.assertEquals(len(data), bytesSent)
sok.close()
acceptCallback()
self.assertEquals('addReader', reactor.calledMethods[1].name)
handleCallback = reactor.calledMethods[1].args[1]
handleCallback()
self.assertEquals('addProcess', reactor.calledMethods[-2].name)
reactor.calledMethods[-2].args[0]()
self.assertEquals(['observer_init', 'handlePacket'],
observer.calledMethodNames())
self.assertEquals(data, observer.calledMethods[1].kwargs['data'])
self.assertEquals('removeReader', reactor.calledMethods[-2].name)
self.assertEquals('removeProcess', reactor.calledMethods[-1].name)
def testServicePrefixSearch(self):
observer = CallTrace('lucene')
def prefixSearch(**kwargs):
raise StopIteration(LuceneResponse(total=2, hits=['aap', 'noot']))
yield
observer.methods['prefixSearch'] = prefixSearch
service = LuceneRemoteService(CallTrace('reactor'))
service.addObserver(observer)
body = dumps({
'message': 'prefixSearch',
'kwargs': {
'prefix': 'aap',
'fieldname': 'field',
'limit': 10,
}
})
result = ''.join(
compose(
service.handleRequest(path='/__lucene_remote__',
Method="POST",
Body=body)))
header, body = result.split('\r\n' * 2)
self.assertTrue('Content-Type: application/json' in header, header)
response = LuceneResponse.fromJson(body)
self.assertEquals(2, response.total)
self.assertEquals(['aap', 'noot'], response.hits)
self.assertEquals(['prefixSearch'], observer.calledMethodNames())
m = observer.calledMethods[0]
self.assertEquals('aap', m.kwargs['prefix'])
self.assertEquals(10, m.kwargs['limit'])
self.assertEquals('field', m.kwargs['fieldname'])
def setUp(self):
self.tempdir = mkdtemp()
self.target = CallTrace("Target")
self.target.path = self.tempdir
self.target.oaiEnvelope = False
logger = CallTrace("Logger")
collection = None
self.uploader = FileSystemUploader(self.target, logger, collection)
def testServiceExecuteQuery(self):
observer = CallTrace('lucene')
def executeQuery(**kwargs):
raise StopIteration(LuceneResponse(total=2, hits=['aap', 'noot']))
yield
observer.methods['executeQuery'] = executeQuery
service = LuceneRemoteService(CallTrace('reactor'))
service.addObserver(observer)
body = dumps({
'message': 'executeQuery',
'kwargs': {
'cqlAbstractSyntaxTree': {
'__CQL_QUERY__': 'query AND field=value'
},
'start': 0,
'stop': 10,
'facets': [{
'fieldname': 'field',
'maxTerms': 5
}],
'filterQueries': [{
'__CQL_QUERY__': 'query=fiets'
}],
'joinQueries': {
'core1': {
'__CQL_QUERY__': 'query=test'
}
}
}
})
result = ''.join(
compose(
service.handleRequest(path='/__lucene_remote__',
Method="POST",
Body=body)))
header, body = result.split('\r\n' * 2)
self.assertTrue('Content-Type: application/json' in header,
header + body)
response = LuceneResponse.fromJson(body)
self.assertEquals(2, response.total)
self.assertEquals(['aap', 'noot'], response.hits)
self.assertEquals(['executeQuery'], observer.calledMethodNames())
m = observer.calledMethods[0]
self.assertEquals(parseString('query AND field=value'),
m.kwargs['cqlAbstractSyntaxTree'])
self.assertEquals(0, m.kwargs['start'])
self.assertEquals(10, m.kwargs['stop'])
self.assertEquals([{
'fieldname': 'field',
'maxTerms': 5
}], m.kwargs['facets'])
self.assertEquals([parseString('query=fiets')],
m.kwargs['filterQueries'])
self.assertEquals({'core1': parseString('query=test')},
m.kwargs['joinQueries'])
def newDNA(self, **kwargs):
emptyGeneratorMethods = [kwargs.get('message', 'handle')]
self.reactor = CallTrace('Reactor', returnValues={'addTimer': 'TOKEN'})
self.observer = CallTrace('Observer', emptyGeneratorMethods=emptyGeneratorMethods, ignoredAttributes=['observer_init'])
self.pc = PeriodicCall(self.reactor, **kwargs)
self.dna = be((Observable(),
(self.pc,
(self.observer,),
),
))
def testCouldHaveInStorage(self):
inputEvent = fromstring('<document><other/></document>')
interceptor = CallTrace('Interceptor', ignoredAttributes=['getData', 'all_unknown', 'any_unknown', 'call_unknown'], methods={'add': yieldNothing})
storage = CallTrace('Storage', ignoredAttributes=['add', 'all_unknown'])
storage.returnValues['getData'] = '<one/>'
v = createVenturiHelix([], [{'partname': 'one', 'xpath': '/document/one'}], interceptor, storage)
list(compose(v.all.add('identifier', 'document', inputEvent)))
self.assertEqual(['begin', 'add'], [m.name for m in interceptor.calledMethods])
self.assertEqual('<one/>', lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode']))
self.assertEqual(dict(identifier='identifier', name='one'), storage.calledMethods[1].kwargs)
def testReadFromStorageAsString(self):
inputEvent = fromstring('<document/>')
interceptor = CallTrace('Interceptor', ignoredAttributes=['getData', 'all_unknown', 'call_unknown'], methods={'add': yieldNothing})
storage = CallTrace('Storage', ignoredAttributes=['add', 'all_unknown'])
storage.returnValues['getData'] = '<some>this is partone</some>'
v = createVenturiHelix([dict(partname='partone', xpath='/document/part[@name="partone"]/text()', asData=True)], [], interceptor, storage)
list(compose(v.all.add('identifier', 'document', inputEvent)))
self.assertEqual(['begin', 'add'], [m.name for m in interceptor.calledMethods])
self.assertEqual('<some>this is partone</some>', interceptor.calledMethods[1].kwargs['data'])
self.assertEqual(dict(identifier='identifier', name='partone'), storage.calledMethods[1].kwargs)
def testCallWithReturnValue(self):
callTrace = CallTrace()
callTrace.returnValues['simpleCall'] = 'OK'
result = callTrace.simpleCall('argument one', 2)
self.assertEqual('OK', result)
callTrace = CallTrace()
callTrace.methods['simpleCall'] = lambda x: x + 3
result = callTrace.simpleCall(1)
self.assertEqual(4, result)
def setUp(self):
super(OnlyAddDeleteIfChangedTest, self).setUp()
self.observer = CallTrace('storageAndMore',
emptyGeneratorMethods=['add', 'delete'],
returnValues={
'getRecord': CallTrace(),
'getData': 'data'
})
self.top = be(
(Observable(), (OnlyAddDeleteIfChanged(), (self.observer, ))))
def setUp(self):
SeecrTestCase.setUp(self)
ctx = CallTrace('CTX')
tx = Transaction('TX')
tx.locals = {'id': 'iden&tifier'}
tx.name = "tsName"
self.fxf = Fields2SolrDoc("tsName", "fields-partname")
self.fxf.ctx = ctx
self.fxf.ctx.tx = tx
self.observer = CallTrace(methods={'add': add})
self.fxf.addObserver(self.observer)
def testMapping(self):
upload = CallTrace('upload')
upload.id = 'id'
upload.isDeleted = True
mapping = CallTrace()
mapping.returnValues['createUpload'] = upload
self.harvest = OnlineHarvest(self.output)
data = 'file://%s/mocktud/00002.xml' % self._testpath
self.harvest.performMapping(None, data, mapping)
self.assertEqual(['addObserver', 'mappingInfo', 'createUpload', 'createUpload', 'createUpload'], [m.name for m in mapping.calledMethods])
for createUploadMethod in mapping.calledMethods[2:]:
self.assertTrue(createUploadMethod.kwargs['doAsserts'])
def setUp(self, fieldRegistry=FieldRegistry()):
super(LuceneTestCase, self).setUp()
self._javaObjects = self._getJavaObjects()
self._reactor = CallTrace('reactor', methods={'addTimer': lambda seconds, callback: CallTrace('timer')})
self._defaultSettings = LuceneSettings(commitCount=1, commitTimeout=1, fieldRegistry=fieldRegistry)
self.lucene = Lucene(
join(self.tempdir, 'lucene'),
reactor=self._reactor,
settings=self._defaultSettings,
)
self.observer = CallTrace()
self.lucene.addObserver(self.observer)
def testSetNameEscaped(self):
repository = CallTrace(returnValues=dict(
requestUrl='http://example.org'))
observer = CallTrace(returnValues=dict(getAllSets=[('123&abc',
'123 & abc')]),
emptyGeneratorMethods=['oaiWatermark'])
top = be((Observable(), (OaiListSets(repository), (observer, ))))
response = asString(
top.all.listSets(arguments=dict(verb=['ListSets'])))
self.assertTrue(
'<ListSets><set><setSpec>123&abc</setSpec><setName>123 & abc</setName></set></ListSets>'
in response, response)
def testOnlySpecifiedMethods(self):
trace = CallTrace(onlySpecifiedMethods=True)
self.assertRaises(AttributeError, lambda: trace.someMethod())
trace = CallTrace(returnValues={"someMethod": 'result'},
onlySpecifiedMethods=True)
self.assertEqual('result', trace.someMethod())
trace = CallTrace(methods={"someMethod": lambda: "result"},
onlySpecifiedMethods=True)
self.assertEqual("result", trace.someMethod())
trace = CallTrace(emptyGeneratorMethods=["someMethod"],
onlySpecifiedMethods=True)
self.assertEqual(GeneratorType, type(trace.someMethod()))
def testDeleteIsAsynchronous(self):
__callstack_var_tx__ = CallTrace('Transaction')
__callstack_var_tx__.locals={}
observer = CallTrace('observer')
callable = lambda: None
observer.returnValues['delete'] = (f for f in [callable])
v = Venturi()
v.addObserver(observer)
result = list(compose(v.delete(identifier='identifier')))
self.assertEqual([callable], result)
self.assertEqual(['delete'], [m.name for m in observer.calledMethods])
def testNothing(self):
__callstack_var_tx__ = CallTrace('TX')
__callstack_var_tx__.locals = {'id': 'identifier'}
intercept = CallTrace()
fields2OaiRecord = Fields2OaiRecord()
fields2OaiRecord.addObserver(intercept)
def f():
f = yield fields2OaiRecord.beginTransaction()
yield f
tx = next(compose(f()))
tx.addField('set', ('setSpec', 'setName'))
tx.commit()
self.assertEqual(0, len(intercept.calledMethods))
