def testFieldWithMultiLevel(self): multi = MultiLevelDrilldown( {'date':[('yearAndMonth', 2, False), ('year', 2, False)] } ) drilldown = CallTrace('Drilldown') doDrilldownArguments = [] def doDrilldown(bitMatrixRow, fieldNamesAndMaxResults): doDrilldownArguments.append((bitMatrixRow, fieldNamesAndMaxResults)) self.assertEquals(1, len(fieldNamesAndMaxResults)) levelField, levelMax, levelSorted = fieldNamesAndMaxResults[0] if levelField == 'yearAndMonth': raise StopIteration(iter([('yearAndMonth', iter([('2008-01',11),('2008-02',2),('2007-12',1)][:levelMax]))])) else: raise StopIteration(iter([('year', iter([('2008',13),('2003',10),('2007',10)][:levelMax]))])) yield drilldown.drilldown = doDrilldown multi.addObserver(drilldown) result = list(compose(multi.multiLevelDrilldown('bitMatrixRow', ['date']))) self.assertEquals(2, len(doDrilldownArguments)) self.assertEquals(('bitMatrixRow', [('yearAndMonth', 2, False)]), doDrilldownArguments[0]) self.assertEquals(('bitMatrixRow', [('year', 2, False)]), doDrilldownArguments[1]) self.assertEquals(1, len(result)) (inField, realField), termCounts = result[0] self.assertEquals('year', realField) self.assertEquals([('2008',13),('2003',10)], list(termCounts))
def testTwoFieldNamesCalled(self): multi = MultiLevelDrilldown( {'date':[('datelevel2',3, False),('datelevel1', 10, False)], 'genre':[('type', 10, False)] } ) drilldown = CallTrace('Drilldown') doDrilldownArguments = [] def doDrilldown(bitMatrixRow, fieldNamesAndMaxResults): doDrilldownArguments.append((bitMatrixRow, fieldNamesAndMaxResults)) self.assertEquals(1, len(fieldNamesAndMaxResults)) levelField, levelMax, levelSorted = fieldNamesAndMaxResults[0] if 'datelevel2' == levelField: raise StopIteration(iter([('datelevel2', iter([('2008',13),('2007',10)][:levelMax]))])) else: raise StopIteration(iter([('type', iter([('literature',43),('donaldduck',30)][:levelMax]))])) yield drilldown.drilldown = doDrilldown multi.addObserver(drilldown) result = list(compose(multi.multiLevelDrilldown('bitMatrixRow', ['date', 'genre']))) self.assertEquals(2, len(doDrilldownArguments)) self.assertEquals(('bitMatrixRow', [('datelevel2', 3, False)]), doDrilldownArguments[0]) self.assertEquals(('bitMatrixRow', [('type', 10, False)]), doDrilldownArguments[1]) self.assertEquals(2, len(result)) self.assertEquals([('date', 'datelevel2'),('genre', 'type')], [(inField, realField) for (inField, realField), termCounts in result])
def testTcpPacketListener(self): reactor = CallTrace('reactor') observer = CallTrace('observer', emptyGeneratorMethods=['handlePacket']) tcpPacketListener = TcpPacketListener(reactor, port=1234) server = be((Observable(), (tcpPacketListener, (observer,) ) )) list(compose(server.once.observer_init())) self.assertEquals('addReader', reactor.calledMethods[0].name) acceptCallback = reactor.calledMethods[0].args[1] data = "TEST" * 1024 sok = socket() sok.connect(('localhost', 1234)) bytesSent = sok.send(data) self.assertEquals(len(data), bytesSent) sok.close() acceptCallback() self.assertEquals('addReader', reactor.calledMethods[1].name) handleCallback = reactor.calledMethods[1].args[1] handleCallback() self.assertEquals('addProcess', reactor.calledMethods[-2].name) reactor.calledMethods[-2].args[0]() self.assertEquals(['observer_init', 'handlePacket'], observer.calledMethodNames()) self.assertEquals(data, observer.calledMethods[1].kwargs['data']) self.assertEquals('removeReader', reactor.calledMethods[-2].name) self.assertEquals('removeProcess', reactor.calledMethods[-1].name)
def testOne(self): observable = Observable() bitMatrixRow = CallTrace('BitMatrixRow') multi = MultiLevelDrilldown( {'date':[('datelevel1', 10, False)]} ) drilldown = CallTrace('Drilldown') def dd(*args, **kwargs): raise StopIteration(iter([('datelevel1', iter([('2008',13),('2007',10)]))])) yield drilldown.methods['drilldown'] = dd multi.addObserver(drilldown) observable.addObserver(multi) result = list(compose(observable.call.multiLevelDrilldown(bitMatrixRow, ['date']))) self.assertEquals(1, len(drilldown.calledMethods)) drilldownMethod = drilldown.calledMethods[0] self.assertEquals('drilldown', drilldownMethod.name) self.assertEquals((bitMatrixRow, [('datelevel1', 10, False)]), drilldownMethod.args) self.assertEquals(1, len(result)) (inputFieldName, realFieldName), termCounts = result[0] self.assertEquals('date', inputFieldName) self.assertEquals('datelevel1', realFieldName) self.assertEquals([('2008',13),('2007',10)], list(termCounts))
def testRewrite(self): rewrite = MessageRewrite(fromMessage='this_message', toMessage='to_message') observer = CallTrace(emptyGeneratorMethods=['to_message']) tree = be((Observable(), (rewrite, (observer,), ) )) consume(tree.all.this_message(aap='noot')) self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) observer.calledMethods.reset() consume(tree.any.this_message(aap='noot')) self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) del observer.emptyGeneratorMethods[:] observer.calledMethods.reset() tree.call.this_message(aap='noot') self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) observer.calledMethods.reset() tree.do.this_message(aap='noot') self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
def getRecord(identifier, *args, **kwargs): record = CallTrace() record.identifier = identifier record.prefixes = set(['oai_dc']) record.sets = set() record.isDeleted = False return record
def testLogCanReturnCallables(self): observer= CallTrace('observer') observer.returnValues['handleRequest'] = (f for f in ['1', lambda: None,'3']) self.queryLog.addObserver(observer) list(compose(self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/path/sru', otherArg='value'))) self.assertEquals(1, len(open(join(self.tempdir, '2009-11-02-query.log')).readlines()))
def testAllQueryHelpersForSRU(self): index = CallTrace('index') def executeQuery(**kwargs): raise StopIteration(Response(total=3201, hits=[])) yield index.methods['executeQuery'] = executeQuery index.ignoredAttributes.extend(['echoedExtraRequestData', 'extraResponseData', 'all_unknown']) server = be((Observable(), (self.queryLog, (SruParser(), (QueryLogHelperForSru(), (SruHandler(extraRecordDataNewStyle=True), (QueryLogHelperForExecuteCQL(), (index,) ) ) ) ) ), )) ''.join(compose(server.all.handleRequest( path='/path/sru', Client=('11.22.33.44', 8080), arguments={ 'operation': ['searchRetrieve'], 'version': ['1.2'], 'maximumRecords': ['0'], 'query': ['field=value'], }, ))) self.assertEquals('2009-11-02T11:25:37Z 11.22.33.44 0.7K 1.000s 3201hits /path/sru maximumRecords=0&operation=searchRetrieve&query=field%3Dvalue&recordPacking=xml&recordSchema=dc&startRecord=1&version=1.2\n', open(join(self.tempdir, '2009-11-02-query.log')).read())
def setUp(self): SeecrTestCase.setUp(self) self.stateDir = join(self.tempdir, "state") self.logDir = join(self.tempdir, "log") self.domainId = "adomain" makedirs(join(self.stateDir, self.domainId)) repoId1LogDir = join(self.logDir, self.domainId, "invalid", "repoId1") repoId2LogDir = join(self.logDir, self.domainId, "invalid", escapeFilename("repoId/2")) makedirs(repoId1LogDir) makedirs(repoId2LogDir) open(join(repoId1LogDir, "invalidId1"), 'w').write("<diagnostic>ERROR1</diagnostic>") open(join(repoId1LogDir, "invalidId&2"), 'w').write("<diagnostic>ERROR2</diagnostic>") open(join(repoId2LogDir, escapeFilename("invalidId/3")), 'w').write("<diagnostic>ERROR3</diagnostic>") open(join(self.stateDir, self.domainId, "repoId1_invalid.ids"), 'w').write("invalidId1\ninvalidId&2") open(join(self.stateDir, self.domainId, escapeFilename("repoId/2_invalid.ids")), 'w').write("invalidId/3") open(join(self.stateDir, self.domainId, "repoId3_invalid.ids"), 'w').write("") self.status = RepositoryStatus(self.logDir, self.stateDir) observer = CallTrace("HarvesterData") observer.returnValues["getRepositoryGroupIds"] = ["repoGroupId1", "repoGroupId2"] def getRepositoryIds(domainId, repositoryGroupId): if repositoryGroupId == "repoGroupId1": return ["repoId1", "repoId/2"] return ["repoId3", "anotherRepoId"] observer.methods["getRepositoryIds"] = getRepositoryIds def getRepositoryGroupId(domainId, repositoryId): return 'repoGroupId1' if repositoryId in ['repoId1', 'repoId/2'] else 'repoGroupId2' observer.methods["getRepositoryGroupId"] = getRepositoryGroupId self.status.addObserver(observer)
def testAdd(self): class Factory(): def __init__(self, observable, untokenizedFieldnames): self.observable = observable self.untokenizedFieldnames = untokenizedFieldnames def fieldsFor(self, fieldname, value): raise StopIteration([(fieldname, value)]) yield fieldFactory = Factory fieldRegistry = FieldRegistry(drilldownFields=[DrilldownField('drilldown.field')]) index = FieldsListToLuceneDocument(fieldRegistry, untokenizedFieldnames=[], indexFieldFactory=fieldFactory) observer = CallTrace(emptyGeneratorMethods=['addDocument']) index.addObserver(observer) fields = [ ("field1", "value1"), ("field2", "value2"), ("drilldown.field", "a drilldown value"), ("__key__.field", "a key value"), ("__key__.field1", 2), ] consume(index.add(identifier="", fieldslist=fields)) self.assertEquals(['addDocument'], observer.calledMethodNames()) fields = observer.calledMethods[0].kwargs['fields'] self.assertEqual([ {'name': 'field1', 'type': 'TextField', 'value': 'value1'}, {'name': 'field2', 'type': 'TextField', 'value': 'value2'}, {'name': 'drilldown.field', 'type': 'FacetField', 'path': ['a drilldown value']}, {'name': '__key__.field', 'type': 'KeyField', 'value': 'a key value'}, {'name': '__key__.field1', 'type': 'KeyField', 'value': 2}, ], fields)
def testDeleteWithoutOaiEnvelope(self): recordFile = join(self.tempdir, 'id.record') os.system('touch ' + recordFile) self.assertTrue(isfile(recordFile)) self.uploader._filenameFor = lambda *args: recordFile self.target.oaiEnvelope = False repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.id = 'repositoryId' upload = Upload(repository=repository) upload.id = 'id' self.uploader.delete(upload) DELETED_RECORDS = join(self.tempdir, 'deleted_records') self.assertTrue(isfile(DELETED_RECORDS)) self.assertEquals(['id\n'], open(DELETED_RECORDS).readlines()) self.assertFalse(isfile(recordFile)) upload.id = 'second:id' self.uploader.delete(upload) self.assertEquals(['id\n', 'second:id\n'], open(DELETED_RECORDS).readlines())
def testQueryTimeInExtraResponse(self): handler = SruHandler(includeQueryTimes=True) observer = CallTrace('observer', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData']) times = [1, 2.5, 3.5] def timeNow(): return times.pop(0) handler._timeNow = timeNow def executeQuery(**kwargs): response = Response(total=0, hits=[]) response.queryTime=5 raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery handler.addObserver(observer) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') result = "".join(compose(handler.searchRetrieve(sruArguments=arguments, **arguments))) sruResponse = parse(StringIO(result)) extraResponseData = sruResponse.xpath('/srw:searchRetrieveResponse/srw:extraResponseData', namespaces={'srw':"http://www.loc.gov/zing/srw/"})[0] self.assertEqualsWS("""<srw:extraResponseData %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <querytimes xmlns="http://meresco.org/namespace/timing"> <sruHandling>PT2.500S</sruHandling> <sruQueryTime>PT1.500S</sruQueryTime> <index>PT0.005S</index> </querytimes> </srw:extraResponseData>""" % namespaces, lxmltostring(extraResponseData)) queryTimes = lxmltostring(extraResponseData.xpath('//ti:querytimes', namespaces={'ti':"http://meresco.org/namespace/timing"})[0]) assertValid(queryTimes, join(schemasPath, 'timing-20120827.xsd')) self.assertEquals(['executeQuery', 'echoedExtraRequestData', 'extraResponseData', 'handleQueryTimes'], observer.calledMethodNames()) self.assertEquals({'sru': Decimal("2.500"), 'queryTime': Decimal("1.500"), 'index': Decimal("0.005")}, observer.calledMethods[3].kwargs)
def testCollectLogWhenIndexRaisesError(self): handler = SruHandler(enableCollectLog=True) observer = CallTrace('observer', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData', 'additionalDiagnosticDetails']) __callstack_var_logCollector__ = dict() times = [1] def timeNow(): return times.pop(0) handler._timeNow = timeNow def executeQuery(**kwargs): raise Exception('Sorry') yield observer.methods['executeQuery'] = executeQuery handler.addObserver(observer) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') consume(handler.searchRetrieve(sruArguments=arguments, **arguments)) self.assertEquals({ 'sru': { 'arguments': [{ 'startRecord': 11, 'query': 'query', 'recordPacking': 'string', 'maximumRecords': 15, 'recordSchema': 'schema', }], } }, __callstack_var_logCollector__)
def testWithSorting(self): mockData = { 'yearAndMonth': [('2008-01',1),('2008-02',2),('2007-12',11)], 'year': [('2008',13),('2003',10),('2005',9), ('2007', 15)] } drilldown = CallTrace('Drilldown') def doDrilldown(bitMatrixRow, fieldNamesAndMaxResults): levelField, levelMax, levelSorted = fieldNamesAndMaxResults[0] data = mockData[levelField] if levelSorted: data = sorted(data, cmp=lambda (term0, card0), (term1, card1): cmp(card1, card0)) if levelMax > 0: data = data[:levelMax] raise StopIteration(iter([(levelField, iter(data))])) yield drilldown.drilldown = doDrilldown multi = MultiLevelDrilldown({'date':[('yearAndMonth', 2, False), ('year', 3, True)]}) multi.addObserver(drilldown) result = list(compose(multi.multiLevelDrilldown('bitMatrixRow', ['date']))) self.assertEquals([(('date', 'year'), [('2007', 15), ('2008', 13), ('2003', 10)])], result) multi = MultiLevelDrilldown({'date':[('yearAndMonth', 4, False), ('year', 3, False)]}) multi.addObserver(drilldown) result = list(compose(multi.multiLevelDrilldown('bitMatrixRow', ['date']))) self.assertEquals([(('date', 'yearAndMonth'), [('2008-01',1),('2008-02',2),('2007-12',11)])], result)
def testLogAsObservable(self): log = CallTrace('log', onlySpecifiedMethods=True, methods={'log': lambda **kwargs: None}) writer = QueryLogWriter() writer.addObserver(log) writer.writeLog(defaultCollectedLogWithPath('/sru')) self.assertEquals(['log'], log.calledMethodNames()) self.assertEquals(['/sru'], [m.kwargs['path'] for m in log.calledMethods])
def testOneResult(self): observer = CallTrace( returnValues={ 'getRecord': '<item><title>Test Title</title><link>Test Identifier</link><description>Test Description</description></item>', }, ignoredAttributes=['unknown', 'extraResponseData', 'echoedExtraRequestData']) def executeQuery(**kwargs): raise StopIteration(Response(total=1, hits=[Hit(1)])) yield observer.methods['executeQuery'] = executeQuery rss = Rss( title = 'Test title', description = 'Test description', link = 'http://www.example.org', sortKeys = 'date,,1', maximumRecords = '15', ) rss.addObserver(observer) result = asString(rss.handleRequest(RequestURI='/?query=aQuery')) self.assertEqualsWS(RSS % """<item> <title>Test Title</title> <link>Test Identifier</link> <description>Test Description</description> </item>""", result)
class DrilldownQueriesTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.dbdq = DrilldownQueries() self.observer = CallTrace(methods=dict(executeQuery=mockExecuteQuery)) self.dbdq.addObserver(self.observer) def testDrilldownQuery(self): result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a = b']})) self.assertEquals('result', result) self.assertEquals(['executeQuery'], self.observer.calledMethodNames()) executeQueryMethod = self.observer.calledMethods[0] self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries']) self.observer.calledMethods.reset() result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a exact b']})) self.assertEquals('result', result) self.assertEquals(['executeQuery'], self.observer.calledMethodNames()) executeQueryMethod = self.observer.calledMethods[0] self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries']) def testErrorForInvalidFormatDrilldownQuery(self): try: retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a']})) self.fail() except ValueError, e: self.assertEquals('x-drilldown-query format should be field=value', str(e)) self.assertEquals([], self.observer.calledMethodNames())
def createUpload(about=None): repository = CallTrace('repository') repository.id = 'repoId' upload = Upload(repository=repository, oaiResponse=oaiResponse(about=about)) upload.id = 'id' return upload
def testShouldUnsetFlagImmediate(self): i = [0] def addTimer(*args, **kwargs): i[0] = i[0] + 1 return i[0] reactor = CallTrace('reactor', methods=dict(addTimer=addTimer)) registry = ServiceRegistry(reactor, self.tempdir, domainname="zp.example.org") observer = CallTrace('observer') registry.addObserver(observer) identifier = str(uuid4()) registry.updateService(identifier=identifier, type='plein', ipAddress='127.0.0.1', infoport=1234, data={}) registry = ServiceRegistry(reactor, self.tempdir, domainname="zp.example.org") flag = READABLE service = registry.getService(identifier) registry.setFlag(identifier, flag, True) self.assertEqual([], reactor.calledMethodNames()) service = registry.getService(identifier) self.assertFalse(service[flag.name], service) state = registry.getPrivateStateFor(identifier) self.assertTrue(state[flag.name], state) self.assertTrue(state[flag.name + "_goingup"]) registry.setFlag(identifier, flag, False, immediate=True) self.assertEqual([], reactor.calledMethodNames()) service = registry.getService(identifier) self.assertFalse(service[flag.name], service) state = registry.getPrivateStateFor(identifier) self.assertFalse(state[flag.name], state) self.assertFalse(flag.name + "_goingup" in state)
def testSimpleCall(self): callTrace = CallTrace() callTrace.simpleCall() self.assertEquals(1, len(callTrace.calledMethods)) tracedCall = callTrace.calledMethods[0] self.assertEquals('simpleCall', tracedCall.name) self.assertEquals(0, len(tracedCall.args))
def testServiceExecuteQuery(self): observer = CallTrace('lucene') def executeQuery(**kwargs): raise StopIteration(LuceneResponse(total=2, hits=['aap','noot'])) yield observer.methods['executeQuery'] = executeQuery service = LuceneRemoteService(CallTrace('reactor')) service.addObserver(observer) body = dumps({ 'message': 'executeQuery', 'kwargs':{ 'cqlAbstractSyntaxTree': {'__CQL_QUERY__': 'query AND field=value'}, 'start':0, 'stop': 10, 'facets': [{'fieldname': 'field', 'maxTerms':5}], 'filterQueries': [{'__CQL_QUERY__': 'query=fiets'}], 'joinQueries': {'core1': {'__CQL_QUERY__': 'query=test'}} } }) result = ''.join(compose(service.handleRequest(path='/__lucene_remote__', Method="POST", Body=body))) header, body = result.split('\r\n'*2) self.assertTrue('Content-Type: application/json' in header, header+body) response = LuceneResponse.fromJson(body) self.assertEquals(2, response.total) self.assertEquals(['aap', 'noot'], response.hits) self.assertEquals(['executeQuery'], observer.calledMethodNames()) m = observer.calledMethods[0] self.assertEquals(parseString('query AND field=value'), m.kwargs['cqlAbstractSyntaxTree']) self.assertEquals(0, m.kwargs['start']) self.assertEquals(10, m.kwargs['stop']) self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], m.kwargs['facets']) self.assertEquals([parseString('query=fiets')], m.kwargs['filterQueries']) self.assertEquals({'core1': parseString('query=test')}, m.kwargs['joinQueries'])
def testCollectLog(self): handler = SruHandler(enableCollectLog=True) observer = CallTrace('observer', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData']) __callstack_var_logCollector__ = dict() times = [1, 2.5, 3.5] def timeNow(): return times.pop(0) handler._timeNow = timeNow def executeQuery(**kwargs): response = Response(total=0, hits=[]) response.queryTime=5 raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery handler.addObserver(observer) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') consume(handler.searchRetrieve(sruArguments=arguments, **arguments)) self.assertEquals({ 'sru': { 'handlingTime': [Decimal('2.500')], 'queryTime': [Decimal('1.500')], 'indexTime': [Decimal('0.005')], 'numberOfRecords': [0], 'arguments': [{ 'startRecord': 11, 'query': 'query', 'recordPacking': 'string', 'maximumRecords': 15, 'recordSchema': 'schema', }], } }, __callstack_var_logCollector__)
def testDeleteWithOaiEnvelope(self): RECORD_FILENAME = join(self.tempdir, 'id.record') self.uploader._filenameFor = lambda *args: RECORD_FILENAME self.uploader.tznow = lambda: "VANDAAG_EN_NU" self.target.oaiEnvelope = True repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.metadataPrefix = 'oai_dc' repository.baseurl = "http://repository" repository.id = 'repositoryId' upload = Upload(repository=repository, oaiResponse=oaiResponse(identifier='id.record', deleted=True)) self.assertFalse(isfile(RECORD_FILENAME)) self.uploader.delete(upload) self.assertTrue(isfile(RECORD_FILENAME)) self.assertEqualsWS("""<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"> <responseDate>VANDAAG_EN_NU</responseDate> <request verb="GetRecord" metadataPrefix="oai_dc" identifier="id.record">http://repository</request> <GetRecord> <record> <header status="deleted"> <identifier>id.record</identifier> <datestamp>2005-08-29T07:08:09Z</datestamp> </header> </record> </GetRecord> </OAI-PMH>""", open(RECORD_FILENAME).read())
def testResumeOrThrowOnlyOnce(self): # A.k.a. Promise / Future like behaviour. trace = CallTrace("Reactor") def getSuspend(): trace.calledMethods.reset() suspend = Suspend(doNext=trace.doNext) self.assertEquals([], trace.calledMethodNames()) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'suspend'], trace.calledMethodNames()) doNextM, suspendM = trace.calledMethods self.assertEquals(((suspend,), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) trace.calledMethods.reset() return suspend suspend = getSuspend() suspend.resume(response='whatever') self.assertEquals(['whenDone'], trace.calledMethodNames()) trace.calledMethods.reset() # Below no change of result, no side-effects self.assertEquals('whatever', suspend.getResult()) try: suspend.resume(response='DIFFERENT') except AssertionError, e: self.assertEquals('Suspend already settled.', str(e))
def testServicePrefixSearch(self): observer = CallTrace('lucene') def prefixSearch(**kwargs): raise StopIteration(LuceneResponse(total=2, hits=['aap','noot'])) yield observer.methods['prefixSearch'] = prefixSearch service = LuceneRemoteService(CallTrace('reactor')) service.addObserver(observer) body = dumps({ 'message': 'prefixSearch', 'kwargs':{ 'prefix':'aap', 'fieldname': 'field', 'limit': 10, } }) result = ''.join(compose(service.handleRequest(path='/__lucene_remote__', Method="POST", Body=body))) header, body = result.split('\r\n'*2) self.assertTrue('Content-Type: application/json' in header, header) response = LuceneResponse.fromJson(body) self.assertEquals(2, response.total) self.assertEquals(['aap', 'noot'], response.hits) self.assertEquals(['prefixSearch'], observer.calledMethodNames()) m = observer.calledMethods[0] self.assertEquals('aap', m.kwargs['prefix']) self.assertEquals(10, m.kwargs['limit']) self.assertEquals('field', m.kwargs['fieldname'])
def testApacheLog(self): requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"]) requestHandler.returnValues["handleRequest"] = (f for f in [Yield, okPlainText, "te", callable, "xt"]) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be( (Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),))) ) result = asList( observable.all.handleRequest( Method="GET", Client=("127.0.0.1", 1234), RequestURI="http://example.org/path?key=value", query="key=value", path="/path", Headers={"Referer": "http://meresco.org", "User-Agent": "Meresco-Components Test"}, otherKwarg="value", ) ) self.assertEquals([Yield, okPlainText, "te", callable, "xt"], result) self.assertEquals(["handleRequest"], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n', logline, )
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{'fieldname': 'field', 'maxTerms':5}], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testUnusedTimeoutSetInitialisesTimer(self): # Whitebox (unusedTimeout -> addTimer) mockReactor = CallTrace() SocketPool(reactor=mockReactor, unusedTimeout=0.02) self.assertEquals(['addTimer'], mockReactor.calledMethodNames()) self.assertEquals(['seconds', 'callback'], mockReactor.calledMethods[0].kwargs.keys()) self.assertEquals(0.02, mockReactor.calledMethods[0].kwargs['seconds']) # Blackbox def test(): top = be((Observable(), (SocketPool(reactor=reactor(), unusedTimeout=0.02),), )) yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('A')) yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('B')) yield sleep(seconds=0.001) result = yield top.any.getPooledSocket(host='x', port=80) self.assertEquals('B', result) yield sleep(seconds=0.04) result = yield top.any.getPooledSocket(host='x', port=80) self.assertEquals(None, result) asProcess(test())
def testLoggedPathsNewStyle(self): log = CallTrace('log') def handleRequest(**kwargs): yield okPlainText yield 'result' index = CallTrace('index', methods={'handleRequest':handleRequest}) observable = be((Observable(), (LogCollector(), (QueryLogWriter(log=log, scopeNames=('global', 'yesPath')),), (LogCollectorScope('global'), (HandleRequestLog(), (PathFilter('/yes'), (LogCollectorScope('yesPath'), (index,), ) ), (PathFilter('/no'), (index,), ) ) ) ) )) result = asString(observable.all.handleRequest(Client=('11.22.33.44', 1234), path='/yes')) self.assertEquals(okPlainText+'result', result) result = asString(observable.all.handleRequest(Client=('22.33.44.55', 2345), path='/no')) self.assertEquals(okPlainText+'result', result) result = asString(observable.all.handleRequest(Client=('33.44.55.66', 3456), path='/yes')) self.assertEquals(okPlainText+'result', result) self.assertEquals(['log', 'log'], log.calledMethodNames()) self.assertEquals(['/yes', '/yes'], [m.kwargs['path'] for m in log.calledMethods])
def getFilename(anId): repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.id = 'repositoryId' upload = Upload(repository=repository) upload.id = anId return self.uploader._filenameFor(upload)
def testCouldHaveInStorage(self): inputEvent = fromstring('<document><other/></document>') interceptor = CallTrace('Interceptor', ignoredAttributes=['getData', 'all_unknown', 'any_unknown', 'call_unknown'], methods={'add': yieldNothing}) storage = CallTrace('Storage', ignoredAttributes=['add', 'all_unknown']) storage.returnValues['getData'] = '<one/>' v = createVenturiHelix([], [{'partname': 'one', 'xpath': '/document/one'}], interceptor, storage) list(compose(v.all.add('identifier', 'document', inputEvent))) self.assertEqual(['begin', 'add'], [m.name for m in interceptor.calledMethods]) self.assertEqual('<one/>', lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode'])) self.assertEqual(dict(identifier='identifier', name='one'), storage.calledMethods[1].kwargs)
def setUp(self): SeecrTestCase.setUp(self) self.client = CallTrace('Client') self.report = AdminServicesReport() self.dna = be((Observable(), (self.report, (self.client,), ) ))
def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace('observer') self.response = Response(total=0, hits=[]) def executeQuery(**kwargs): return self.response yield self.observer.methods['executeQuery'] = executeQuery
def testAddHeaders(self): handler = BasicHttpHandler(additionalHeaders={'Aap': "Noot Mies", "Boom": "Vis"}) observer = CallTrace('HttpComponent') observer.returnValues['handleRequest'] = (f for f in ['HT','TP/1.0 200 OK\r\n\r\n', 'Body']) dna = self.build(handler, observer) response = generatorToString(dna.all.handleRequest(RequestURI="/")) self.assertEqual('HTTP/1.0 200 OK\r\nAap: Noot Mies\r\nBoom: Vis\r\n\r\nBody', response) self.assertEqual(['handleRequest'], observer.calledMethodNames())
def testXmlPrintLxmlPrettyPrintFalse(self): observable = Observable() xmlprintlxml = XmlPrintLxml(fromKwarg='lxmlNode', toKwarg="data", pretty_print=False) observer = CallTrace('observer', emptyGeneratorMethods=['someMessage']) xmlprintlxml.addObserver(observer) observable.addObserver(xmlprintlxml) list(compose(observable.all.someMessage(lxmlNode=parse(StringIO('<a><b>“c</b></a>'))))) self.assertEquals(['someMessage'], observer.calledMethodNames()) self.assertEquals(['data'], observer.calledMethods[0].kwargs.keys()) self.assertEquals('''<a><b>“c</b></a>''', observer.calledMethods[0].kwargs['data'])
def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace() with stderr_replaced() as err: self.dna = be( (Observable(), (OaiSetSelect(['set1', 'set2']), (self.observer, )))) self.assertTrue(not err.getvalue() or \ 'warn("OaiSetSelect is deprecated;' in err.getvalue(), err.getvalue())
def testCouldHaveButDoesnot(self): inputEvent = fromstring('<document><other/></document>') interceptor = CallTrace('Interceptor', ignoredAttributes=['getData', 'all_unknown', 'any_unknown', 'call_unknown'], methods={'add': yieldNothing}) storage = CallTrace('Storage', ignoredAttributes=['add', 'all_unknown']) storage.exceptions['getData'] = KeyError('Part not available') v = createVenturiHelix([{'partname': 'other', 'xpath': '/document/other'}], [{'partname': 'one', 'xpath': '/document/one'}], interceptor, storage) list(compose(v.all.add('identifier', 'document', inputEvent))) self.assertEqual(['begin', 'add'], [m.name for m in interceptor.calledMethods]) self.assertEqual('identifier', interceptor.calledMethods[1].kwargs['identifier']) self.assertEqual('other', interceptor.calledMethods[1].kwargs['partname'])
def testOk(self): handler = BasicHttpHandler() observer = CallTrace('HttpComponent') observer.returnValues['handleRequest'] = (f for f in ['HTTP/1.0 200 OK\r\n\r\n', 'Body']) dna = self.build(handler, observer) response = generatorToString(dna.all.handleRequest(RequestURI="/")) self.assertEqual('HTTP/1.0 200 OK\r\n\r\nBody', response) self.assertEqual(['handleRequest'], observer.calledMethodNames())
def testReadFromStorageAsString(self): inputEvent = fromstring('<document/>') interceptor = CallTrace('Interceptor', ignoredAttributes=['getData', 'all_unknown', 'call_unknown'], methods={'add': yieldNothing}) storage = CallTrace('Storage', ignoredAttributes=['add', 'all_unknown']) storage.returnValues['getData'] = '<some>this is partone</some>' v = createVenturiHelix([dict(partname='partone', xpath='/document/part[@name="partone"]/text()', asData=True)], [], interceptor, storage) list(compose(v.all.add('identifier', 'document', inputEvent))) self.assertEqual(['begin', 'add'], [m.name for m in interceptor.calledMethods]) self.assertEqual('<some>this is partone</some>', interceptor.calledMethods[1].kwargs['data']) self.assertEqual(dict(identifier='identifier', name='partone'), storage.calledMethods[1].kwargs)
def testDeleteAlsoSetsIdOnTransaction(self): __callstack_var_tx__ = CallTrace('Transaction') __callstack_var_tx__.locals = {} v = Venturi(should=[{ 'partname': 'PARTNAME', 'xpath': '/document' }], could=[]) list(compose(v.delete(identifier='identifier'))) self.assertEquals('identifier', __callstack_var_tx__.locals['id'])
def newDNA(self, **kwargs): emptyGeneratorMethods = [kwargs.get('message', 'handle')] self.reactor = CallTrace('Reactor', returnValues={'addTimer': 'TOKEN'}) self.observer = CallTrace('Observer', emptyGeneratorMethods=emptyGeneratorMethods, ignoredAttributes=['observer_init']) self.pc = PeriodicCall(self.reactor, **kwargs) self.dna = be((Observable(), (self.pc, (self.observer,), ), ))
def setUp(self): super(OnlyAddDeleteIfChangedTest, self).setUp() self.observer = CallTrace('storageAndMore', emptyGeneratorMethods=['add', 'delete'], returnValues={ 'getRecord': CallTrace(), 'getData': 'data' }) self.top = be( (Observable(), (OnlyAddDeleteIfChanged(), (self.observer, ))))
def testWebQueryIgnoresWrongFilters(self): observer = CallTrace( ignoredAttributes=['unknown', 'extraResponseData', 'echoedExtraRequestData']) observer.exceptions['executeQuery'] = StopIteration([0, []]) rss = Rss(title = 'Title', description = 'Description', link = 'Link') rss.addObserver(observer) result = "".join(rss.handleRequest(RequestURI='/?query=one+two&filter=invalid&filter=')) self.assertTrue("<description>An error occurred 'Invalid filter: invalid'</description>" in result, result)
def testRemoteExecuteQueryWithNoneValues(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse( total=5, hits=[Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator( observable.any.executeQuery( cqlAbstractSyntaxTree=parseString('query AND field=value'), start=0, stop=10, facets=None, filterQueries=None, joinQueries=None, )) self.assertEquals(5, result.total) self.assertEquals( [Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) self.assertDictEquals( { 'message': 'executeQuery', 'kwargs': { 'cqlAbstractSyntaxTree': { '__CQL_QUERY__': 'query AND field=value' }, 'start': 0, 'stop': 10, 'facets': None, 'filterQueries': None, 'joinQueries': None, } }, loads(m.kwargs['body']))
def testGetRecordWithRepositoryIdentifier(self): oaigetrecord = OaiGetRecord(OaiRepository(identifier='example.org')) record = CallTrace('record') record.identifier = 'id0' record.prefixes = ['oai_dc'] record.sets = [] record.isDeleted = False observer = CallTrace( returnValues={ 'isKnownPrefix': True, 'getRecord': record }, emptyGeneratorMethods=['oaiWatermark', 'oaiRecord']) oaigetrecord.addObserver(observer) consume( oaigetrecord.getRecord(arguments=dict( verb=['GetRecord'], metadataPrefix=['oai_dc'], identifier=['oai:example.org:id0'], ), **self.httpkwargs)) self.assertEqual( ['getRecord', 'isKnownPrefix', 'oaiWatermark', 'oaiRecord'], observer.calledMethodNames()) self.assertEqual(dict(identifier='id0', metadataPrefix='oai_dc'), observer.calledMethods[0].kwargs)
def testLoggedPathsIsStartOfAcceptedPath(self): observer = CallTrace('observer') observer.returnValues['handleRequest'] = (line for line in ['1', '2', '3']) self.queryLog.addObserver(observer) ''.join( compose( self.queryLog.handleRequest(Client=('127.0.0.1', 47785), path='/path/sru/extended/path', otherArg='value'))) self.assertEqual(1, len(listdir(self.tempdir)))
def setUp(self): SeecrTestCase.setUp(self) ctx = CallTrace('CTX') tx = Transaction('TX') tx.locals = {'id': 'iden&tifier'} tx.name = "tsName" self.fxf = Fields2SolrDoc("tsName", "fields-partname") self.fxf.ctx = ctx self.fxf.ctx.tx = tx self.observer = CallTrace(methods={'add': add}) self.fxf.addObserver(self.observer)
def testLogForNumberOfRecordsSelection(self): log = CallTrace('log') writer = QueryLogWriter(log=log, numberOfRecordsSelection=dict(scope='myscope', key='total')) collectedLog = defaultCollectedLog() collectedLog['myscope'] = {'total': [100]} writer.writeLog(collectedLog) self.assertEquals(['log'], log.calledMethodNames()) self.assertEquals( [100], [m.kwargs['numberOfRecords'] for m in log.calledMethods])
def testCallWithException(self): class TestException(Exception): pass callTrace = CallTrace() callTrace.exceptions['simpleCall'] = TestException('test') try: result = callTrace.simpleCall() self.fail() except TestException as e: self.assertEqual('test', str(e))
def testAddDocument(self): fields2LuceneDoc = Fields2LuceneDoc('tsname', fieldRegistry=FieldRegistry()) observer = CallTrace() fields2LuceneDoc.addObserver(observer) fields2LuceneDoc.ctx.tx = Transaction('tsname') fields2LuceneDoc.ctx.tx.locals['id'] = 'identifier' fields2LuceneDoc.addField('field', 'value') consume(fields2LuceneDoc.commit('unused')) self.assertEquals(['addDocument'], observer.calledMethodNames()) self.assertEquals('identifier', observer.calledMethods[0].kwargs['identifier'])
def testChangePasswordWrongOld(self): observer = CallTrace() self.form.addObserver(observer) observer.returnValues['validateUser'] = False Body = urlencode(dict(username='******', oldPassword='******', newPassword="******", retypedPassword="******", formUrl='/show/changepasswordform')) session = {'user': BasicHtmlLoginForm.User('user')} result = asString(self.form.handleRequest(path='/login/changepassword', Client=('127.0.0.1', 3451), Method='POST', Body=Body, session=session)) self.assertEquals({'username':'******', 'errorMessage': 'Username and password do not match'}, session['BasicHtmlLoginForm.formValues']) self.assertEquals("HTTP/1.0 302 Found\r\nLocation: /show/changepasswordform\r\n\r\n", result)
def testConnect(self): reactor = CallTrace() acceptor = Acceptor(reactor, self.port, lambda sok: None) self.assertEquals('addReader', reactor.calledMethods[0].name) acceptCallback = reactor.calledMethods[0].args[1] sok = socket() sok.connect(('localhost', self.port)) acceptCallback() self.assertEquals(['addReader'], reactor.calledMethodNames()) reactor.calledMethods[0].args[0].close() sok.close()
def testChangePassword(self): observer = CallTrace() self.form.addObserver(observer) observer.returnValues['validateUser'] = True Body = urlencode(dict( username='******', oldPassword='******', newPassword="******", retypedPassword="******", formUrl='/show/changepasswordform', returnUrl='/home')) session = {'user': BasicHtmlLoginForm.User('user')} result = asString(self.form.handleRequest(path='/login/changepassword', Client=('127.0.0.1', 3451), Method='POST', Body=Body, session=session)) self.assertEquals(['validateUser', 'setPassword'], [m.name for m in observer.calledMethods]) self.assertEquals("HTTP/1.0 302 Found\r\nLocation: /home\r\n\r\n", result)
def testMappingRaisesException(self): mapping = CallTrace() self.harvest = OnlineHarvest(self.output) mapping.exceptions['createUpload'] = Exception('Mushroom, mushroom') data = 'file://%s/mocktud/00002.xml' % self._testpath try: self.harvest.performMapping(None, data, mappingObject=mapping) self.fail() except Exception as ex: self.assertEqual('Mushroom, mushroom', str(ex)) self.assertEqual('\n',self.output.getvalue())
def setUp(self, fieldRegistry=FieldRegistry()): super(LuceneTestCase, self).setUp() self._javaObjects = self._getJavaObjects() self._reactor = CallTrace('reactor', methods={'addTimer': lambda seconds, callback: CallTrace('timer')}) self._defaultSettings = LuceneSettings(commitCount=1, commitTimeout=1, fieldRegistry=fieldRegistry) self.lucene = Lucene( join(self.tempdir, 'lucene'), reactor=self._reactor, settings=self._defaultSettings, ) self.observer = CallTrace() self.lucene.addObserver(self.observer)
def testGetData(self): adapter = StorageAdapter() observer = CallTrace(returnValues=dict(getStream=StringIO("DATA"))) adapter.addObserver(observer) result = adapter.getData(identifier='id:1', name='partname') self.assertEqual('DATA', result) self.assertEqual(['getStream'], observer.calledMethodNames()) self.assertEqual({ 'partname': 'partname', 'identifier': 'id:1' }, observer.calledMethods[0].kwargs)
def testDetectValidUserWithPasswordAndUserName(self): authentication = BasicAuthentication(realm='Test Realm') interceptor = CallTrace('httphandler', returnValues={'isValidLogin': True}, methods={'handleRequest': lambda *a, **kw: (x for x in 'response')}) authentication.addObserver(interceptor) headers = {'Authorization': 'Basic ' + b64encode('aUser:aPassword')} results = authentication.handleRequest(port='8080', RequestURI='/private', Method='GET', Headers=headers) response = ''.join(compose(results)) self.assertFalse('WWW-Authenticate: Basic realm="Test Realm"\r\n' in response, response) interceptor.returnValues['isValidLogin'] = False headers = {'Authorization': 'Basic ' + b64encode('aUser:aCompletelyWrongPassword')} response = ''.join(authentication.handleRequest(port='8080', RequestURI='/private', Method='GET', Headers=headers)) self.assertTrue('WWW-Authenticate: Basic realm="Test Realm"\r\n' in response, response)
def testShouldCallPreviouslyRegisteredSignalHandlersAfterHandleShutdown( self): reactor = Reactor() called = [] def handleShutdown(): called.append('handleShutdown') testCoName = currentframe().f_code.co_name def prevIntHandler(signum, frame): self.assertEqual(SIGINT, signum) self.assertEqual(testCoName, frame.f_code.co_name) called.append('prevIntHandler') trace = CallTrace('Observer', methods={'handleShutdown': handleShutdown}) top = be(( Observable(), ( Observable(), # Only once calls walk the observer tree. (trace, ), ), )) origIntHandler = signal(SIGINT, prevIntHandler) try: shutdownHandler = registerShutdownHandler(statePath=self.tempdir, server=top, reactor=reactor) reactor.addTimer(0.01, lambda: None) with stdout_replaced() as output: try: kill(getpid(), SIGINT) reactor.loop() self.fail('should terminate') except KeyboardInterrupt: pass self.assertTrue( 'Scheduled for immediate shutdown.\n' in output.getvalue(), output.getvalue()) self.assertTrue('Shutdown completed.\n' in output.getvalue(), output.getvalue()) finally: shutdownHandler.unregister() signal(SIGINT, origIntHandler) self.assertEqual(['handleShutdown'], trace.calledMethodNames()) self.assertEqual( ((), {}), (trace.calledMethods[0].args, trace.calledMethods[0].kwargs)) self.assertEqual(['handleShutdown', 'prevIntHandler'], called)
def testPausePausesOnStart(self): # autoStart reactor = CallTrace('reactor') pc = PeriodicCall(reactor=reactor, autoStart=False) pc.observer_init() self.assertEquals([], reactor.calledMethodNames()) # explicit .pause() pc = PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True) pc.pause() pc.observer_init() self.assertEquals([], reactor.calledMethodNames())