def testIncrementalHarvestScheduleSetToNone(self): observer = CallTrace(emptyGeneratorMethods=['add']) oaiDownloadProcessor = OaiDownloadProcessor( path="/oai", metadataPrefix="oai_dc", incrementalHarvestSchedule=Schedule(period=0), workingDirectory=self.tempdir, xWait=False, err=StringIO()) oaiDownloadProcessor.addObserver(observer) consume( oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE)))) self.assertEqual('2002-06-01T19:20:30Z', oaiDownloadProcessor._from) self.assertNotEqual(None, oaiDownloadProcessor._earliestNextRequestTime) self.assertEqual( ['startOaiBatch', 'add', 'stopOaiBatch', 'signalHarvestingDone'], observer.calledMethodNames()) observer.calledMethods.reset() oaiDownloadProcessor.setFrom(from_=None) oaiDownloadProcessor.setIncrementalHarvestSchedule(schedule=None) consume( oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE)))) self.assertEqual('2002-06-01T19:20:30Z', oaiDownloadProcessor._from) self.assertEqual(None, oaiDownloadProcessor._earliestNextRequestTime) self.assertEqual( ['startOaiBatch', 'add', 'stopOaiBatch', 'signalHarvestingDone'], observer.calledMethodNames())
def testRewrite(self): rewrite = MessageRewrite(fromMessage='this_message', toMessage='to_message') observer = CallTrace(emptyGeneratorMethods=['to_message']) tree = be((Observable(), (rewrite, (observer,), ) )) consume(tree.all.this_message(aap='noot')) self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) observer.calledMethods.reset() consume(tree.any.this_message(aap='noot')) self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) del observer.emptyGeneratorMethods[:] observer.calledMethods.reset() tree.call.this_message(aap='noot') self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) observer.calledMethods.reset() tree.do.this_message(aap='noot') self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
def testSuspendTimingOut(self): # with calltrace; happy path trace = CallTrace(returnValues={'addTimer': 'timerToken'}) suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) self.assertEquals([], trace.calledMethodNames()) self.assertEquals(None, suspend._timer) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) doNextM, addTimerM, suspendM = trace.calledMethods self.assertEquals(((suspend, ), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), { 'seconds': 3.14, 'callback': suspend._timedOut }), (addTimerM.args, addTimerM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) self.assertEquals('timerToken', suspend._timer) trace.calledMethods.reset() suspend._timedOut() self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) onTimeoutM, whenDoneM = trace.calledMethods self.assertEquals(((), {}), (onTimeoutM.args, onTimeoutM.kwargs)) self.assertEquals(((), {}), (whenDoneM.args, whenDoneM.kwargs)) self.assertEquals(None, suspend._timer) self.assertRaises(TimeoutException, lambda: suspend.getResult())
def testRewrite(self): rewrite = MessageRewrite(fromMessage='this_message', toMessage='to_message') observer = CallTrace(emptyGeneratorMethods=['to_message']) tree = be((Observable(), ( rewrite, (observer, ), ))) consume(tree.all.this_message(aap='noot')) self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) observer.calledMethods.reset() consume(tree.any.this_message(aap='noot')) self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) del observer.emptyGeneratorMethods[:] observer.calledMethods.reset() tree.call.this_message(aap='noot') self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs) observer.calledMethods.reset() tree.do.this_message(aap='noot') self.assertEqual(['to_message'], observer.calledMethodNames()) self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
class DrilldownQueriesTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.dbdq = DrilldownQueries() self.observer = CallTrace(methods=dict(executeQuery=mockExecuteQuery)) self.dbdq.addObserver(self.observer) def testDrilldownQuery(self): result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a = b']})) self.assertEquals('result', result) self.assertEquals(['executeQuery'], self.observer.calledMethodNames()) executeQueryMethod = self.observer.calledMethods[0] self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries']) self.observer.calledMethods.reset() result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a exact b']})) self.assertEquals('result', result) self.assertEquals(['executeQuery'], self.observer.calledMethodNames()) executeQueryMethod = self.observer.calledMethods[0] self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries']) def testErrorForInvalidFormatDrilldownQuery(self): try: retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a']})) self.fail() except ValueError, e: self.assertEquals('x-drilldown-query format should be field=value', str(e)) self.assertEquals([], self.observer.calledMethodNames())
def testShouldUnsetFlagImmediate(self): i = [0] def addTimer(*args, **kwargs): i[0] = i[0] + 1 return i[0] reactor = CallTrace('reactor', methods=dict(addTimer=addTimer)) registry = ServiceRegistry(reactor, self.tempdir, domainname="zp.example.org") observer = CallTrace('observer') registry.addObserver(observer) identifier = str(uuid4()) registry.updateService(identifier=identifier, type='plein', ipAddress='127.0.0.1', infoport=1234, data={}) registry = ServiceRegistry(reactor, self.tempdir, domainname="zp.example.org") flag = READABLE service = registry.getService(identifier) registry.setFlag(identifier, flag, True) self.assertEqual([], reactor.calledMethodNames()) service = registry.getService(identifier) self.assertFalse(service[flag.name], service) state = registry.getPrivateStateFor(identifier) self.assertTrue(state[flag.name], state) self.assertTrue(state[flag.name + "_goingup"]) registry.setFlag(identifier, flag, False, immediate=True) self.assertEqual([], reactor.calledMethodNames()) service = registry.getService(identifier) self.assertFalse(service[flag.name], service) state = registry.getPrivateStateFor(identifier) self.assertFalse(state[flag.name], state) self.assertFalse(flag.name + "_goingup" in state)
class RememberMeCookieTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.paths = [] def handleRequest(path, *args, **kwargs): self.paths.append(path) yield "RESPONSE" def validateMethod(cookie): return dict( value="USER", header="COOKIE HEADER" ) if cookie == "THIS IS THE REMEMBER ME COOKIE" else None self.observer = CallTrace( methods={ 'handleRequest': handleRequest, 'validateCookie': validateMethod, 'cookieName': lambda: "CID" }) self.dna = be((Observable(), (RememberMeCookie(), (self.observer, )))) def testNoCookie(self): session = {} response = asString( self.dna.all.handleRequest(path='/some_page', Headers={}, session=session)) self.assertEqual("RESPONSE", response) self.assertEqual(['/some_page'], self.paths) self.assertFalse('user' in session, session) self.assertEqual(['cookieName', 'handleRequest'], self.observer.calledMethodNames()) self.assertEquals({ 'path': '/some_page', 'session': {}, 'Headers': {} }, self.observer.calledMethods[-1].kwargs) def testCookie(self): session = {} Headers = dict(Cookie="CID=THIS IS THE REMEMBER ME COOKIE") response = asString( self.dna.all.handleRequest(path='/some_page', Headers=Headers, session=session)) self.assertEqual("RESPONSE", response) self.assertEqual(['/some_page'], self.paths) self.assertTrue('user' in session, session) self.assertEqual(['cookieName', 'validateCookie', 'handleRequest'], self.observer.calledMethodNames()) self.assertEquals( { 'path': '/some_page', 'session': { 'user': '******' }, 'Headers': Headers }, self.observer.calledMethods[-1].kwargs)
def testAddDelete(self): observer = CallTrace('observer') registry = ObjectRegistry(self.tempdir, name='name', redirectPath='/redirect') registry.addObserver(observer) registry.registerKeys(keys=['name', 'key2', 'another'], booleanKeys=['enabled', 'feature']) object1id = registry.addObject(name=["object1"], key2=["value_1"], enabled=['on']) object2id = registry.addObject(name=["object2"], key2=["value_2"], enabled=['on']) self.assertEquals( { object1id: { 'key2': 'value_1', 'enabled': True, 'name': 'object1', 'another': '', 'feature': False }, object2id: { 'key2': 'value_2', 'enabled': True, 'name': 'object2', 'another': '', 'feature': False } }, registry.listObjects()) self.assertEquals(['objectAdded', 'objectAdded'], observer.calledMethodNames()) self.assertEquals({ 'name': 'name', 'identifier': object1id }, observer.calledMethods[0].kwargs) observer.calledMethods.reset() registry.removeObject(identifier=object2id) self.assertEquals(['objectRemoved'], observer.calledMethodNames()) self.assertEquals({ 'name': 'name', 'identifier': object2id }, observer.calledMethods[0].kwargs) registry = ObjectRegistry(self.tempdir, name='name', redirectPath='/redirect') self.assertEquals( { object1id: { 'key2': 'value_1', 'enabled': True, 'name': 'object1', 'another': '', 'feature': False }, }, registry.listObjects())
class OaiSetMaskTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace() self.dna = be((Observable(), (OaiSetMask(['set1', 'set2'], name="set1|set2 mask"), (self.observer, )))) def testOaiSelect(self): self.dna.call.oaiSelect() self.assertEqual(1, len(self.observer.calledMethods)) self.assertEqual(set(['set1', 'set2']), self.observer.calledMethods[0].kwargs['setsMask']) def testOaiSelectWithSetsMask(self): self.dna.call.oaiSelect(setsMask=['set3']) self.assertEqual(1, len(self.observer.calledMethods)) self.assertEqual(set(['set1', 'set2', 'set3']), self.observer.calledMethods[0].kwargs['setsMask']) def testGetUniqueInSets(self): self.observer.returnValues['getRecord'] = MockRecord( 'id', sets=['set1', 'set2', 'set3']) self.dna.call.getRecord('xyz') self.assertEqual(['getRecord'], self.observer.calledMethodNames()) getRecordCall = self.observer.calledMethods[0] self.assertEqual(('xyz', ), getRecordCall.args) def testGetUniqueNotInSets(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1']) self.dna.call.getRecord('xyz') self.assertEqual(['getRecord'], self.observer.calledMethodNames()) def testGetUniqueWithSetsMask(self): self.observer.returnValues['getRecord'] = MockRecord( 'id', sets=['set1', 'set2', 'set3']) self.dna.call.getRecord('xyz', setsMask=['set3']) self.assertEqual(['getRecord'], self.observer.calledMethodNames()) getRecordCall = self.observer.calledMethods[0] self.assertEqual(('xyz', ), getRecordCall.args) self.observer.calledMethods.reset() self.observer.returnValues['getRecord'] = MockRecord( 'id', sets=['set1', 'set2']) self.dna.call.getRecord('xyz', setsMask=['set3']) self.assertEqual(['getRecord'], self.observer.calledMethodNames()) def testOtherMethodsArePassed(self): self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: ( x for x in []) list(compose(self.dna.all.getAllMetadataFormats())) self.assertEqual(1, len(self.observer.calledMethods)) self.assertEqual('getAllMetadataFormats', self.observer.calledMethods[0].name)
class OaiPmhDownloadTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.log = StringIO() self.oaiListRequest = CallTrace() self.oaiListRequest.returnValues['create'] = self.oaiListRequest self.oaiListRequest.returnValues['buildUrl'] = 'http://example.org/oai?verb=ListRequest' self.oaiBatch = CallTrace() self.oaiListRequest.returnValues['retrieveBatch'] = self.oaiBatch self.oaiBatch.request = self.oaiBatch self.oaiBatch.baseurl = 'http://base.example.org' self.oaiBatch.resumptionToken = 'continueHere' self.record = CallTrace() self.record.identifier = 'identifier' self.oaiBatch.items = [self.record] def prepareDownload(self, repositories, **kwargs): dl = OaiPmhDownload(repositories=repositories, log=self.log, **kwargs) dl._OaiListRequest = self.oaiListRequest.create return dl def testNoRepositories(self): dl = self.prepareDownload(repositories=[]) dl.downloadBatch({}) self.assertEquals([], self.oaiListRequest.calledMethodNames()) def testOneRepository(self): repo1 = { 'baseurl': 'http://example.org/oai', 'metadataPrefix': 'prefix', 'repositoryGroupId': 'group' } dl = self.prepareDownload(repositories=[repo1]) batch = dl.downloadBatch({}) self.assertEquals(['create', 'buildUrl', 'retrieveBatch'], self.oaiListRequest.calledMethodNames()) self.assertEquals({'baseurl': 'http://example.org/oai', 'metadataPrefix': 'prefix', 'set': None}, self.oaiListRequest.calledMethods[0].kwargs) self.assertEquals({'repositoriesRemaining': [repo1], 'resumptionToken': 'continueHere'}, batch.resumptionAttributes()) self.assertEquals(["group:identifier"], [r.identifier for r in batch.records]) self.oaiListRequest.calledMethods.reset() batch = dl.downloadBatch(batch.resumptionAttributes()) self.assertEquals(['create', 'buildUrl', 'retrieveBatch'], self.oaiListRequest.calledMethodNames()) self.assertEquals({'baseurl': 'http://example.org/oai', 'resumptionToken':'continueHere'}, self.oaiListRequest.calledMethods[0].kwargs) def testOneRepositoryFinishing(self): self.oaiBatch.resumptionToken = None repo1 = { 'baseurl': 'http://example.org/oai', 'metadataPrefix': 'prefix', 'repositoryGroupId': 'group' } dl = self.prepareDownload(repositories=[repo1]) batch = dl.downloadBatch({}) self.assertEquals({'repositoriesRemaining': [], 'resumptionToken': None}, batch.resumptionAttributes()) self.assertEquals('prefix', batch.metadataPrefix)
def testLogLiveExample(self): collectedLog = { 'httpRequest': { 'timestamp': [1396596372.708574], 'Headers': [{}], 'Client': [('127.0.0.1', 57075)], 'arguments': [{ 'query': ['meta.upload.id exact "NICL:oai:mdms.kenict.org:oai:nicl.nl:k163645"'], 'operation': ['searchRetrieve'], 'version': ['1.2'], 'recordPacking': ['xml'], 'recordSchema': ['smbAggregatedData'] }], 'RequestURI': ['/edurep/sruns?query=meta.upload.id+exact+%22NICL%3Aoai%3Amdms.kenict.org%3Aoai%3Anicl.nl%3Ak163645%22&operation=searchRetrieve&version=1.2&recordPacking=xml&recordSchema=smbAggregatedData'], 'query': ['query=meta.upload.id+exact+%22NICL%3Aoai%3Amdms.kenict.org%3Aoai%3Anicl.nl%3Ak163645%22&operation=searchRetrieve&version=1.2&recordPacking=xml&recordSchema=smbAggregatedData'], 'path': ['/edurep/sruns'], 'Method': ['GET'], 'HTTPVersion': ['1.0'] }, 'query-scope': { 'sub-scope': { 'cqlClauses': [2], 'sru': { 'indexTime': [Decimal('0.000')], 'handlingTime': [Decimal('0.004')], 'numberOfRecords': [1], 'queryTime': [Decimal('0.003')], 'arguments': [{ 'recordSchema': 'smbAggregatedData', 'version': '1.2', 'recordPacking': 'xml', 'maximumRecords': 10, 'startRecord': 1, 'query': 'meta.upload.id exact "NICL:oai:mdms.kenict.org:oai:nicl.nl:k163645"', 'operation': 'searchRetrieve' }] } } }, 'httpResponse': { 'duration': [0.004216909408569336], 'httpStatus': ['200'], 'size': [1889] } } log = CallTrace('log') writer = QueryLogWriter(log=log, scopeNames=('query-scope', 'sub-scope')) log2 = CallTrace('log') writer2 = QueryLogWriter(log=log2, scopeNames=('query-scope', 'other-scope')) writer.writeLog(collectedLog) writer2.writeLog(collectedLog) self.assertEquals(['log'], log.calledMethodNames()) self.assertEquals([], log2.calledMethodNames()) self.assertEquals(['maximumRecords=10&operation=searchRetrieve&query=meta.upload.id+exact+%22NICL%3Aoai%3Amdms.kenict.org%3Aoai%3Anicl.nl%3Ak163645%22&recordPacking=xml&recordSchema=smbAggregatedData&startRecord=1&version=1.2'], [m.kwargs['queryArguments'] for m in log.calledMethods])
def testPausePausesOnStart(self): # autoStart reactor = CallTrace('reactor') pc = PeriodicCall(reactor=reactor, autoStart=False) pc.observer_init() self.assertEquals([], reactor.calledMethodNames()) # explicit .pause() pc = PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True) pc.pause() pc.observer_init() self.assertEquals([], reactor.calledMethodNames())
def prepare(): trace = CallTrace(returnValues={'addTimer': 'timerToken'}) suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) trace.calledMethods.reset() suspend._timedOut() self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) self.assertEquals(True, suspend._settled) trace.calledMethods.reset() return trace, suspend
class OaiSetMaskTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace() self.dna = be( (Observable(), (OaiSetMask(['set1', 'set2'], name="set1|set2 mask"), (self.observer,) ) ) ) def testOaiSelect(self): self.dna.call.oaiSelect() self.assertEquals(1, len(self.observer.calledMethods)) self.assertEquals(set(['set1', 'set2']), self.observer.calledMethods[0].kwargs['setsMask']) def testOaiSelectWithSetsMask(self): self.dna.call.oaiSelect(setsMask=['set3']) self.assertEquals(1, len(self.observer.calledMethods)) self.assertEquals(set(['set1', 'set2', 'set3']), self.observer.calledMethods[0].kwargs['setsMask']) def testGetUniqueInSets(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1', 'set2', 'set3']) self.dna.call.getRecord('xyz') self.assertEquals(['getRecord'], self.observer.calledMethodNames()) getRecordCall = self.observer.calledMethods[0] self.assertEquals(('xyz',), getRecordCall.args) def testGetUniqueNotInSets(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1']) self.dna.call.getRecord('xyz') self.assertEquals(['getRecord'], self.observer.calledMethodNames()) def testGetUniqueWithSetsMask(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1', 'set2', 'set3']) self.dna.call.getRecord('xyz', setsMask=['set3']) self.assertEquals(['getRecord'], self.observer.calledMethodNames()) getRecordCall = self.observer.calledMethods[0] self.assertEquals(('xyz',), getRecordCall.args) self.observer.calledMethods.reset() self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1', 'set2']) self.dna.call.getRecord('xyz', setsMask=['set3']) self.assertEquals(['getRecord'], self.observer.calledMethodNames()) def testOtherMethodsArePassed(self): self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: (x for x in []) list(compose(self.dna.all.getAllMetadataFormats())) self.assertEquals(1, len(self.observer.calledMethods)) self.assertEquals('getAllMetadataFormats', self.observer.calledMethods[0].name)
def testAdditionalArguments(self): log = CallTrace('log') writer = QueryLogWriter(log=log) observer = CallTrace('additional', returnValues={'determineQueryArguments': dict(key='value')}) writer.addObserver(observer) writer.writeLog(defaultCollectedLog()) self.assertEquals(['log'], log.calledMethodNames()) self.assertEquals(['key=value'], [m.kwargs['queryArguments'] for m in log.calledMethods]) self.assertEquals(['determineQueryArguments'], observer.calledMethodNames()) self.assertEquals(dict( collectedLog=defaultCollectedLog(), scopeNames=(), currentArgs={'version': '1.2'}, ), observer.calledMethods[0].kwargs)
def prepare(): trace = CallTrace(returnValues={'addTimer': 'timerToken'}) suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) self.assertEquals([], trace.calledMethodNames()) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) doNextM, addTimerM, suspendM = trace.calledMethods self.assertEquals(((suspend,), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), {'seconds': 3.14, 'callback': suspend._timedOut}), (addTimerM.args, addTimerM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) trace.calledMethods.reset() return trace, suspend
def testObservableCall(self): o = Observable() calltrace = CallTrace('calltrace') o.addObserver(calltrace) result = o.call.getSomething() self.assertEqual(None, result) self.assertEquals(['getSomething'], calltrace.calledMethodNames()) calltrace.calledMethods.reset() calltrace.returnValues['getSomething'] = 'RESULT' result = o.call.getSomething() self.assertEqual('RESULT', result) self.assertEquals(['getSomething'], calltrace.calledMethodNames())
class CombinePartsTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.combine = CombineParts({'together':['one', 'two']}) self.observer = CallTrace('observer') self.combine.addObserver(self.observer) self.observer.methods['getData'] = lambda identifier, name: '<%s/>' % name def testPassThroughOtherStuff(self): result = self.combine.getData(identifier='identifier', name='name') self.assertEquals('<name/>', result) self.assertEquals(['getData'], self.observer.calledMethodNames()) self.assertEquals([dict(identifier='identifier', name='name')], [m.kwargs for m in self.observer.calledMethods]) def testTogether(self): result = self.combine.getData(identifier='identifier', name='together') expected = '<doc:document xmlns:doc="http://meresco.org/namespace/harvester/document"><doc:part name="one"><one/></doc:part><doc:part name="two"><two/></doc:part></doc:document>' self.assertEquals(expected, result) self.assertEquals(['getData', 'getData'], self.observer.calledMethodNames()) self.assertEquals([dict(identifier='identifier', name='one'), dict(identifier='identifier', name='two')], [m.kwargs for m in self.observer.calledMethods]) def testTogetherWithOnePartMissingAllowed(self): self.combine = CombineParts({'together':['one', 'two']}, allowMissingParts=['two']) self.combine.addObserver(self.observer) def getData(identifier, name): if name == 'two': raise KeyError('two') return '<%s/>' % name self.observer.methods['getData'] = getData result = self.combine.getData(identifier='identifier', name='together') self.assertEquals('<doc:document xmlns:doc="http://meresco.org/namespace/harvester/document"><doc:part name="one"><one/></doc:part></doc:document>', result) def testTogetherWithOnePartMissingNotAllowed(self): def getData(identifier, name): if name == 'two': raise KeyError('two') return '<%s/>' % name self.observer.methods['getData'] = getData self.assertRaises(KeyError, lambda: self.combine.getData(identifier='identifier', name='together')) def testTogetherWithGivenMissingPartsAllowed(self): self.combine = CombineParts({'together':['one', 'two']}, allowMissingParts=['two']) self.combine.addObserver(self.observer) def getData(identifier, name): if name == 'one': raise KeyError('one') return '<%s/>' % name self.observer.methods['getData'] = getData self.assertRaises(KeyError, lambda: self.combine.getData(identifier='identifier', name='together'))
class OaiSetSelectTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace() with stderr_replaced() as err: self.dna = be( (Observable(), (OaiSetSelect(['set1', 'set2']), (self.observer, )))) self.assertTrue(not err.getvalue() or \ 'warn("OaiSetSelect is deprecated;' in err.getvalue(), err.getvalue()) def testOaiSelect(self): self.dna.call.oaiSelect() self.assertEqual(1, len(self.observer.calledMethods)) methodCalled = self.observer.calledMethods[0] self.assertTrue('sets' in methodCalled.kwargs, methodCalled) self.assertEqual(['set1', 'set2'], self.observer.calledMethods[0].kwargs['sets']) def testGetUniqueInSet(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1']) self.dna.call.getRecord('xyz') self.assertEqual(['getRecord'], self.observer.calledMethodNames()) getRecordCall = self.observer.calledMethods[0] self.assertEqual(('xyz', ), getRecordCall.args) def testGetUniqueNotInSet(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set4']) self.dna.call.getRecord('xyz') self.assertEqual(['getRecord'], self.observer.calledMethodNames()) def testOtherMethodsArePassed(self): self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: ( x for x in []) list(compose(self.dna.all.getAllMetadataFormats())) self.assertEqual(1, len(self.observer.calledMethods)) self.assertEqual('getAllMetadataFormats', self.observer.calledMethods[0].name) def testSetsIsNone(self): self.dna.call.oaiSelect(sets=None) self.assertEqual(1, len(self.observer.calledMethods)) methodCalled = self.observer.calledMethods[0] self.assertTrue('sets' in methodCalled.kwargs, methodCalled) self.assertEqual(['set1', 'set2'], self.observer.calledMethods[0].kwargs['sets'])
def testAutomatic(self): def handleRequest(**kwargs): yield 'DATA' observer = CallTrace(methods=dict(handleRequest=handleRequest)) vo = SruVersionAndOperation(version='8.7', recordSchema='this') vo.addObserver(observer) result = asString( vo.handleRequest(arguments={ 'query': ['query'], 'recordSchema': ['schema'] })) self.assertEquals('DATA', result) self.assertEqual(['handleRequest'], observer.calledMethodNames()) self.assertEqual( { 'arguments': { 'query': ['query'], 'version': ['8.7'], 'operation': ['searchRetrieve'], 'recordSchema': ['schema'], } }, observer.calledMethods[0].kwargs) observer.calledMethods.reset() result = asString( vo.handleRequest(arguments={ 'query': ['query'], 'version': ['1.2'] })) self.assertEquals('DATA', result) self.assertEqual(['handleRequest'], observer.calledMethodNames()) self.assertEqual( { 'arguments': { 'query': ['query'], 'version': ['1.2'], 'operation': ['searchRetrieve'], 'recordSchema': ['this'], } }, observer.calledMethods[0].kwargs) observer.calledMethods.reset() result = asString(vo.handleRequest(arguments={'noquery': ['noquery']})) self.assertEqual({'arguments': { 'noquery': ['noquery'], }}, observer.calledMethods[0].kwargs)
class OaiSetSelectTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace() with stderr_replaced() as err: self.dna = be( (Observable(), (OaiSetSelect(['set1', 'set2']), (self.observer,) ) ) ) self.assertTrue(not err.getvalue() or \ 'warn("OaiSetSelect is deprecated;' in err.getvalue(), err.getvalue()) def testOaiSelect(self): self.dna.call.oaiSelect() self.assertEquals(1, len(self.observer.calledMethods)) methodCalled = self.observer.calledMethods[0] self.assertTrue('sets' in methodCalled.kwargs, methodCalled) self.assertEquals(['set1', 'set2'], self.observer.calledMethods[0].kwargs['sets']) def testGetUniqueInSet(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1']) self.dna.call.getRecord('xyz') self.assertEquals(['getRecord'], self.observer.calledMethodNames()) getRecordCall = self.observer.calledMethods[0] self.assertEquals(('xyz',), getRecordCall.args) def testGetUniqueNotInSet(self): self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set4']) self.dna.call.getRecord('xyz') self.assertEquals(['getRecord'], self.observer.calledMethodNames()) def testOtherMethodsArePassed(self): self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: (x for x in []) list(compose(self.dna.all.getAllMetadataFormats())) self.assertEquals(1, len(self.observer.calledMethods)) self.assertEquals('getAllMetadataFormats', self.observer.calledMethods[0].name) def testSetsIsNone(self): self.dna.call.oaiSelect(sets=None) self.assertEquals(1, len(self.observer.calledMethods)) methodCalled = self.observer.calledMethods[0] self.assertTrue('sets' in methodCalled.kwargs, methodCalled) self.assertEquals(['set1', 'set2'], self.observer.calledMethods[0].kwargs['sets'])
class MultiCqlToLuceneQueryTest(TestCase): def setUp(self): coreAConverter = CqlToLuceneQuery([('fieldA', 1.0)], luceneSettings=LuceneSettings()) coreBConverter = CqlToLuceneQuery([('fieldB', 1.0)], luceneSettings=LuceneSettings()) self.converter = MultiCqlToLuceneQuery(defaultCore='A', coreToCqlLuceneQueries=dict( A=coreAConverter, B=coreBConverter)) self.observer = CallTrace('Query responder', methods={'executeQuery': executeQueryMock}) self.dna = be((Observable(), ( self.converter, (self.observer, ), ))) def testConvertComposedQuery(self): q = ComposedQuery('A') q.setCoreQuery(core='A', query=parseString('valueAQ')) q.setCoreQuery(core='B', query=parseString('valueBQ')) q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB')) q.addUnite(dict(core='A', query=parseString('fieldUA exact valueUA')), dict(core='B', query=parseString('fieldUB exact valueUB'))) q.validate() consume(self.dna.any.executeComposedQuery(query=q)) self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames()) self.assertEquals(repr(TermQuery(Term('fieldA', 'valueaq'))), repr(q.queryFor('A'))) self.assertEquals(repr(TermQuery(Term('fieldB', 'valuebq'))), repr(q.queryFor('B')))
def testResumeOrThrowOnlyOnce(self): # A.k.a. Promise / Future like behaviour. trace = CallTrace("Reactor") def getSuspend(): trace.calledMethods.reset() suspend = Suspend(doNext=trace.doNext) self.assertEquals([], trace.calledMethodNames()) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'suspend'], trace.calledMethodNames()) doNextM, suspendM = trace.calledMethods self.assertEquals(((suspend,), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) trace.calledMethods.reset() return suspend suspend = getSuspend() suspend.resume(response='whatever') self.assertEquals(['whenDone'], trace.calledMethodNames()) trace.calledMethods.reset() # Below no change of result, no side-effects self.assertEquals('whatever', suspend.getResult()) try: suspend.resume(response='DIFFERENT') except AssertionError, e: self.assertEquals('Suspend already settled.', str(e))
def testLoggedPathsNewStyle(self): log = CallTrace('log') def handleRequest(**kwargs): yield okPlainText yield 'result' index = CallTrace('index', methods={'handleRequest':handleRequest}) observable = be((Observable(), (LogCollector(), (QueryLogWriter(log=log, scopeNames=('global', 'yesPath')),), (LogCollectorScope('global'), (HandleRequestLog(), (PathFilter('/yes'), (LogCollectorScope('yesPath'), (index,), ) ), (PathFilter('/no'), (index,), ) ) ) ) )) result = asString(observable.all.handleRequest(Client=('11.22.33.44', 1234), path='/yes')) self.assertEquals(okPlainText+'result', result) result = asString(observable.all.handleRequest(Client=('22.33.44.55', 2345), path='/no')) self.assertEquals(okPlainText+'result', result) result = asString(observable.all.handleRequest(Client=('33.44.55.66', 3456), path='/yes')) self.assertEquals(okPlainText+'result', result) self.assertEquals(['log', 'log'], log.calledMethodNames()) self.assertEquals(['/yes', '/yes'], [m.kwargs['path'] for m in log.calledMethods])
def testUpdateObject(self): observer = CallTrace('observer') registry = ObjectRegistry(self.tempdir, name='name', redirectPath='/redirect') registry.addObserver(observer) registry.registerKeys(keys=['key2', 'name'], booleanKeys=['enabled']) object1id = registry.addObject(name=["object1"], key2=["value_1"], enabled=['on']) registry.updateObject(identifier=object1id, name=["object1"], key2=["value_2"]) self.assertEquals(['objectAdded', 'objectUpdated'], observer.calledMethodNames()) self.assertEquals({ 'name': 'name', 'identifier': object1id }, observer.calledMethods[-1].kwargs) registry = ObjectRegistry(self.tempdir, name='name', redirectPath='/redirect') self.assertEquals( { object1id: { 'key2': 'value_2', 'enabled': False, 'name': 'object1' } }, registry.listObjects())
class FlagCheckTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.flagCheck = FlagCheck(serviceIdentifier='identifier', flag=READABLE) def handleRequest(**kwargs): yield 'HTTP/1.0 200 Ok\r\nContent-Type: plain/text\r\n\r\nRESULT' self.observer = CallTrace(emptyGeneratorMethods=['someMessage'], methods={'handleRequest':handleRequest}) self.server = be((Observable(), (self.flagCheck, (self.observer,), ) )) def testAllowedWhenFlagSet(self): consume(self.flagCheck.updateConfig(this_service={'readable': False, 'state':{'readable': True}})) consume(self.server.all.someMessage(ignored='ignored')) self.assertEquals(['someMessage'], self.observer.calledMethodNames()) def testNotAllowedServiceDoesntExists(self): consume(self.flagCheck.updateConfig(this_service=None)) try: consume(self.server.all.someMessage(ignored='ignored')) self.fail() except EnvironmentError, e: self.assertEquals("'someMessage' is not allowed at the moment (readable=False).", str(e))
def testApacheLog(self): requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"]) requestHandler.returnValues["handleRequest"] = (f for f in [Yield, okPlainText, "te", callable, "xt"]) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be( (Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),))) ) result = asList( observable.all.handleRequest( Method="GET", Client=("127.0.0.1", 1234), RequestURI="http://example.org/path?key=value", query="key=value", path="/path", Headers={"Referer": "http://meresco.org", "User-Agent": "Meresco-Components Test"}, otherKwarg="value", ) ) self.assertEquals([Yield, okPlainText, "te", callable, "xt"], result) self.assertEquals(["handleRequest"], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n', logline, )
def testRemotePrefixSearch(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse( total=5, hits=["1", "2", "3", "4", "5"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator( observable.any.prefixSearch(prefix='aap', fieldname='field', limit=10)) self.assertEquals(5, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals( { 'message': 'prefixSearch', 'kwargs': { 'prefix': 'aap', 'fieldname': 'field', 'limit': 10, } }, loads(m.kwargs['body']))
def testReadFromStorage(self): inputEvent = fromstring('<document/>') interceptor = CallTrace('Interceptor', ignoredAttributes=[ 'getData', 'all_unknown', 'any_unknown', 'call_unknown' ]) interceptor.methods['add'] = yieldNothing storage = CallTrace('Storage', ignoredAttributes=['add', 'all_unknown']) storage.returnValues['getData'] = '<some>this is partone</some>' v = createVenturiHelix( [{ 'partname': 'partone', 'xpath': '/document/part[@name="partone"]/text()' }], [], interceptor, storage) list(compose(v.all.add('identifier', 'document', inputEvent))) self.assertEquals(['begin', 'add'], [m.name for m in interceptor.calledMethods]) self.assertEquals( '<some>this is partone</some>', lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode'])) self.assertEquals(['begin', 'getData'], storage.calledMethodNames()) self.assertEquals(dict(identifier='identifier', name='partone'), storage.calledMethods[1].kwargs)
def testServicePrefixSearch(self): observer = CallTrace('lucene') def prefixSearch(**kwargs): raise StopIteration(LuceneResponse(total=2, hits=['aap', 'noot'])) yield observer.methods['prefixSearch'] = prefixSearch service = LuceneRemoteService(CallTrace('reactor')) service.addObserver(observer) body = dumps({ 'message': 'prefixSearch', 'kwargs': { 'prefix': 'aap', 'fieldname': 'field', 'limit': 10, } }) result = ''.join( compose( service.handleRequest(path='/__lucene_remote__', Method="POST", Body=body))) header, body = result.split('\r\n' * 2) self.assertTrue('Content-Type: application/json' in header, header) response = LuceneResponse.fromJson(body) self.assertEquals(2, response.total) self.assertEquals(['aap', 'noot'], response.hits) self.assertEquals(['prefixSearch'], observer.calledMethodNames()) m = observer.calledMethods[0] self.assertEquals('aap', m.kwargs['prefix']) self.assertEquals(10, m.kwargs['limit']) self.assertEquals('field', m.kwargs['fieldname'])
def testLogAsObservable(self): log = CallTrace('log', onlySpecifiedMethods=True, methods={'log': lambda **kwargs: None}) writer = QueryLogWriter() writer.addObserver(log) writer.writeLog(defaultCollectedLogWithPath('/sru')) self.assertEquals(['log'], log.calledMethodNames()) self.assertEquals(['/sru'], [m.kwargs['path'] for m in log.calledMethods])
def testResumeOrThrowOnlyOnce(self): # A.k.a. Promise / Future like behaviour. trace = CallTrace("Reactor") def getSuspend(): trace.calledMethods.reset() suspend = Suspend(doNext=trace.doNext) self.assertEquals([], trace.calledMethodNames()) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'suspend'], trace.calledMethodNames()) doNextM, suspendM = trace.calledMethods self.assertEquals(((suspend, ), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) trace.calledMethods.reset() return suspend suspend = getSuspend() suspend.resume(response='whatever') self.assertEquals(['whenDone'], trace.calledMethodNames()) trace.calledMethods.reset() # Below no change of result, no side-effects self.assertEquals('whatever', suspend.getResult()) try: suspend.resume(response='DIFFERENT') except AssertionError, e: self.assertEquals('Suspend already settled.', str(e))
def testLogRequestInCaseOfExceptionAfterStatusCode(self): requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown']) def handleRequestRaisesException(**kwargs): yield Yield yield okPlainText yield 'text' raise ValueError('doesntreallymatter') yield requestHandler.methods['handleRequest'] = handleRequestRaisesException stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), (LogCollector(), (handleRequestLog, (requestHandler,) ), (ApacheLogWriter(stream),), ) )) result = [] g = compose(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test'}, otherKwarg='value')) try: for x in g: result.append(x) except Exception: pass self.assertEqual([Yield, bytes(okPlainText, encoding='utf-8'), b'text'], result) self.assertEqual(['handleRequest'], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\')\n', logline)
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{ 'fieldname': 'field', 'maxTerms': 5 }], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator( observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{ 'fieldname': 'field', 'maxTerms': 5 }], query.facetsFor('coreA'))
def testTcpPacketListener(self): reactor = CallTrace('reactor') observer = CallTrace('observer', emptyGeneratorMethods=['handlePacket']) tcpPacketListener = TcpPacketListener(reactor, port=1234) server = be((Observable(), (tcpPacketListener, (observer, )))) list(compose(server.once.observer_init())) self.assertEquals('addReader', reactor.calledMethods[0].name) acceptCallback = reactor.calledMethods[0].args[1] data = "TEST" * 1024 sok = socket() sok.connect(('localhost', 1234)) bytesSent = sok.send(data) self.assertEquals(len(data), bytesSent) sok.close() acceptCallback() self.assertEquals('addReader', reactor.calledMethods[1].name) handleCallback = reactor.calledMethods[1].args[1] handleCallback() self.assertEquals('addProcess', reactor.calledMethods[-2].name) reactor.calledMethods[-2].args[0]() self.assertEquals(['observer_init', 'handlePacket'], observer.calledMethodNames()) self.assertEquals(data, observer.calledMethods[1].kwargs['data']) self.assertEquals('removeReader', reactor.calledMethods[-2].name) self.assertEquals('removeProcess', reactor.calledMethods[-1].name)
class CountReportTest(SeecrTestCase): def testCountOne(self): self.report = CallTrace(returnValues={'report': None}, onlySpecifiedMethods=True) self.top = be((Observable(), (LogCollector(), ( GustosLogWriter(), (CountReport(gustosGroup='gustosGroup', scopeNames=(), curveName='calls'), ), (CountReport(gustosGroup='gustosGroup', scopeNames=(), curveName='collectTimes', keys=('collectResponse', 'some')), ), (self.report, ), ), (CollectMeALog(), )))) self.top.do.callMe(times=3) self.assertEqual(['report'], self.report.calledMethodNames()) countReport = self.report.calledMethods[0].kwargs['values'][ 'gustosGroup']['Counts'] self.assertEqual({ 'collectTimes': { 'count': 3 }, 'calls': { 'count': 1 } }, countReport)
def testUnusedTimeoutSetInitialisesTimer(self): # Whitebox (unusedTimeout -> addTimer) mockReactor = CallTrace() SocketPool(reactor=mockReactor, unusedTimeout=0.02) self.assertEquals(['addTimer'], mockReactor.calledMethodNames()) self.assertEquals(['seconds', 'callback'], mockReactor.calledMethods[0].kwargs.keys()) self.assertEquals(0.02, mockReactor.calledMethods[0].kwargs['seconds']) # Blackbox def test(): top = be((Observable(), (SocketPool(reactor=reactor(), unusedTimeout=0.02),), )) yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('A')) yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('B')) yield sleep(seconds=0.001) result = yield top.any.getPooledSocket(host='x', port=80) self.assertEquals('B', result) yield sleep(seconds=0.04) result = yield top.any.getPooledSocket(host='x', port=80) self.assertEquals(None, result) asProcess(test())
def testAdd(self): class Factory(): def __init__(self, observable, untokenizedFieldnames): self.observable = observable self.untokenizedFieldnames = untokenizedFieldnames def fieldsFor(self, fieldname, value): raise StopIteration([(fieldname, value)]) yield fieldFactory = Factory fieldRegistry = FieldRegistry(drilldownFields=[DrilldownField('drilldown.field')]) index = FieldsListToLuceneDocument(fieldRegistry, untokenizedFieldnames=[], indexFieldFactory=fieldFactory) observer = CallTrace(emptyGeneratorMethods=['addDocument']) index.addObserver(observer) fields = [ ("field1", "value1"), ("field2", "value2"), ("drilldown.field", "a drilldown value"), ("__key__.field", "a key value"), ("__key__.field1", 2), ] consume(index.add(identifier="", fieldslist=fields)) self.assertEquals(['addDocument'], observer.calledMethodNames()) fields = observer.calledMethods[0].kwargs['fields'] self.assertEqual([ {'name': 'field1', 'type': 'TextField', 'value': 'value1'}, {'name': 'field2', 'type': 'TextField', 'value': 'value2'}, {'name': 'drilldown.field', 'type': 'FacetField', 'path': ['a drilldown value']}, {'name': '__key__.field', 'type': 'KeyField', 'value': 'a key value'}, {'name': '__key__.field1', 'type': 'KeyField', 'value': 2}, ], fields)
def testDeleteWithError(self): ids = Ids(self.tmp_path / "test.ids") try: for i in range(10): ids.add('id:{}'.format(i)) deleteIds = DeleteIds(Bucket(id='test')) def delete(anUpload): if anUpload.id == 'id:7': raise ValueError('fout') observer = CallTrace( methods=dict(delete=delete, deleteIdentifier=ids.remove, flushIds=lambda **kwargs: ids.reopen())) deleteIds.addObserver(observer) self.assertRaises(ValueError, lambda: deleteIds.delete(ids)) self.assertEqual(['id:7', 'id:8', 'id:9'], readIds(join(self.tempdir, 'test.ids'))) self.assertEqual( 8, len([m for m in observer.calledMethodNames() if m == 'delete'])) finally: ids.close()
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{'fieldname': 'field', 'maxTerms':5}], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testUnusedTimeoutSetInitialisesTimer(self): # Whitebox (unusedTimeout -> addTimer) mockReactor = CallTrace() SocketPool(reactor=mockReactor, unusedTimeout=0.02) self.assertEquals(['addTimer'], mockReactor.calledMethodNames()) self.assertEquals(['seconds', 'callback'], mockReactor.calledMethods[0].kwargs.keys()) self.assertEquals(0.02, mockReactor.calledMethods[0].kwargs['seconds']) # Blackbox def test(): top = be(( Observable(), (SocketPool(reactor=reactor(), unusedTimeout=0.02), ), )) yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('A')) yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('B')) yield sleep(seconds=0.001) result = yield top.any.getPooledSocket(host='x', port=80) self.assertEquals('B', result) yield sleep(seconds=0.04) result = yield top.any.getPooledSocket(host='x', port=80) self.assertEquals(None, result) asProcess(test())
def testServiceExecuteQuery(self): observer = CallTrace('lucene') def executeQuery(**kwargs): raise StopIteration(LuceneResponse(total=2, hits=['aap','noot'])) yield observer.methods['executeQuery'] = executeQuery service = LuceneRemoteService(CallTrace('reactor')) service.addObserver(observer) body = dumps({ 'message': 'executeQuery', 'kwargs':{ 'cqlAbstractSyntaxTree': {'__CQL_QUERY__': 'query AND field=value'}, 'start':0, 'stop': 10, 'facets': [{'fieldname': 'field', 'maxTerms':5}], 'filterQueries': [{'__CQL_QUERY__': 'query=fiets'}], 'joinQueries': {'core1': {'__CQL_QUERY__': 'query=test'}} } }) result = ''.join(compose(service.handleRequest(path='/__lucene_remote__', Method="POST", Body=body))) header, body = result.split('\r\n'*2) self.assertTrue('Content-Type: application/json' in header, header+body) response = LuceneResponse.fromJson(body) self.assertEquals(2, response.total) self.assertEquals(['aap', 'noot'], response.hits) self.assertEquals(['executeQuery'], observer.calledMethodNames()) m = observer.calledMethods[0] self.assertEquals(parseString('query AND field=value'), m.kwargs['cqlAbstractSyntaxTree']) self.assertEquals(0, m.kwargs['start']) self.assertEquals(10, m.kwargs['stop']) self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], m.kwargs['facets']) self.assertEquals([parseString('query=fiets')], m.kwargs['filterQueries']) self.assertEquals({'core1': parseString('query=test')}, m.kwargs['joinQueries'])
def testServicePrefixSearch(self): observer = CallTrace('lucene') def prefixSearch(**kwargs): raise StopIteration(LuceneResponse(total=2, hits=['aap','noot'])) yield observer.methods['prefixSearch'] = prefixSearch service = LuceneRemoteService(CallTrace('reactor')) service.addObserver(observer) body = dumps({ 'message': 'prefixSearch', 'kwargs':{ 'prefix':'aap', 'fieldname': 'field', 'limit': 10, } }) result = ''.join(compose(service.handleRequest(path='/__lucene_remote__', Method="POST", Body=body))) header, body = result.split('\r\n'*2) self.assertTrue('Content-Type: application/json' in header, header) response = LuceneResponse.fromJson(body) self.assertEquals(2, response.total) self.assertEquals(['aap', 'noot'], response.hits) self.assertEquals(['prefixSearch'], observer.calledMethodNames()) m = observer.calledMethods[0] self.assertEquals('aap', m.kwargs['prefix']) self.assertEquals(10, m.kwargs['limit']) self.assertEquals('field', m.kwargs['fieldname'])
class ResponseTimeReportTest(SeecrTestCase): def testCountOne(self): self.report = CallTrace(returnValues={'report': { "packet": "data" }}, onlySpecifiedMethods=True) self.top = be((Observable(), (LogCollector(), ( GustosLogWriter(), (ResponseTimeReport(gustosGroup='gustosGroup', scopeNames=(), curveName='http'), ), (ResponseTimeReport(gustosGroup='gustosGroup', scopeNames=(), curveName='other', keys=('otherResponse', 'duration')), ), (self.report, ), ), (CollectMeALog(), )))) self.top.do.callMe(times=3) self.assertEqual(['report'], self.report.calledMethodNames()) theReport = self.report.calledMethods[0].kwargs['values'][ 'gustosGroup']['ResponseTime'] self.assertEqual({ 'other': { 'time': 0.3 }, 'http': { 'time': 0.0 } }, theReport)
def testTcpPacketListener(self): reactor = CallTrace('reactor') observer = CallTrace('observer', emptyGeneratorMethods=['handlePacket']) tcpPacketListener = TcpPacketListener(reactor, port=1234) server = be((Observable(), (tcpPacketListener, (observer,) ) )) list(compose(server.once.observer_init())) self.assertEquals('addReader', reactor.calledMethods[0].name) acceptCallback = reactor.calledMethods[0].args[1] data = "TEST" * 1024 sok = socket() sok.connect(('localhost', 1234)) bytesSent = sok.send(data) self.assertEquals(len(data), bytesSent) sok.close() acceptCallback() self.assertEquals('addReader', reactor.calledMethods[1].name) handleCallback = reactor.calledMethods[1].args[1] handleCallback() self.assertEquals('addProcess', reactor.calledMethods[-2].name) reactor.calledMethods[-2].args[0]() self.assertEquals(['observer_init', 'handlePacket'], observer.calledMethodNames()) self.assertEquals(data, observer.calledMethods[1].kwargs['data']) self.assertEquals('removeReader', reactor.calledMethods[-2].name) self.assertEquals('removeProcess', reactor.calledMethods[-1].name)
def testApacheLog(self): requestHandler = CallTrace( 'handler', ignoredAttributes=['writeLog', 'do_unknown']) requestHandler.returnValues['handleRequest'] = ( f for f in [Yield, okPlainText, 'te', callable, 'xt']) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), ( LogCollector(), (handleRequestLog, (requestHandler, )), (ApacheLogWriter(stream), ), ))) result = asList( observable.all.handleRequest( Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={ 'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test' }, otherKwarg='value')) self.assertEquals([Yield, okPlainText, 'te', callable, 'xt'], result) self.assertEquals(['handleRequest'], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n', logline)
def testGetRecordWithRepositoryIdentifier(self): oaigetrecord = OaiGetRecord(OaiRepository(identifier='example.org')) record = CallTrace('record') record.identifier = 'id0' record.prefixes = ['oai_dc'] record.sets = [] record.isDeleted = False observer = CallTrace( returnValues={ 'isKnownPrefix': True, 'getRecord': record }, emptyGeneratorMethods=['oaiWatermark', 'oaiRecord']) oaigetrecord.addObserver(observer) consume( oaigetrecord.getRecord(arguments=dict( verb=['GetRecord'], metadataPrefix=['oai_dc'], identifier=['oai:example.org:id0'], ), **self.httpkwargs)) self.assertEqual( ['getRecord', 'isKnownPrefix', 'oaiWatermark', 'oaiRecord'], observer.calledMethodNames()) self.assertEqual(dict(identifier='id0', metadataPrefix='oai_dc'), observer.calledMethods[0].kwargs)
class DrilldownQueriesTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.dbdq = DrilldownQueries() self.observer = CallTrace(methods=dict(executeQuery=mockExecuteQuery)) self.dbdq.addObserver(self.observer) def testDrilldownQuery(self): result = retval( self.dbdq.executeQuery( extraArguments={'x-drilldown-query': ['a = b']})) self.assertEqual('result', result) self.assertEqual(['executeQuery'], self.observer.calledMethodNames()) executeQueryMethod = self.observer.calledMethods[0] self.assertEqual([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries']) self.observer.calledMethods.reset() result = retval( self.dbdq.executeQuery( extraArguments={'x-drilldown-query': ['a exact b']})) self.assertEqual('result', result) self.assertEqual(['executeQuery'], self.observer.calledMethodNames()) executeQueryMethod = self.observer.calledMethods[0] self.assertEqual([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries']) def testErrorForInvalidFormatDrilldownQuery(self): try: retval( self.dbdq.executeQuery( extraArguments={'x-drilldown-query': ['a']})) self.fail() except ValueError as e: self.assertEqual('x-drilldown-query format should be field=value', str(e)) self.assertEqual([], self.observer.calledMethodNames()) def testNoDrilldownQuery(self): result = retval(self.dbdq.executeQuery(extraArguments={}, query='*')) self.assertEqual('result', result) self.assertEqual(['executeQuery'], self.observer.calledMethodNames()) executeQueryMethod = self.observer.calledMethods[0] self.assertEqual([], executeQueryMethod.kwargs['drilldownQueries']) self.assertEqual("*", executeQueryMethod.kwargs['query'])
def testLogForNumberOfRecordsSelection(self): log = CallTrace('log') writer = QueryLogWriter(log=log, numberOfRecordsSelection=dict(scope='myscope', key='total')) collectedLog = defaultCollectedLog() collectedLog['myscope'] = {'total': [100]} writer.writeLog(collectedLog) self.assertEquals(['log'], log.calledMethodNames()) self.assertEquals([100], [m.kwargs['numberOfRecords'] for m in log.calledMethods])
def testLogForArgumentsInsteadOfSruArguments(self): log = CallTrace('log') writer = QueryLogWriter.forHttpArguments(log=log) collectedLog = defaultCollectedLog() collectedLog['httpRequest']['arguments'] = [{'verb':'ListRecords', 'metadataPrefix':'rdf'}] writer.writeLog(collectedLog) self.assertEquals(['log'], log.calledMethodNames()) self.assertEquals(['metadataPrefix=rdf&verb=ListRecords'], [m.kwargs['queryArguments'] for m in log.calledMethods])
def testOne(self): normdocToFieldsList = NormdocToFieldsList() observer = CallTrace(emptyGeneratorMethods=['add']) normdocToFieldsList.addObserver(observer) consume(normdocToFieldsList.add(identifier='record:4', lxmlNode=parse(open("data/normdoc.xml")) )) self.assertEqual(['add'], observer.calledMethodNames()) # print "FIELDLIST:", observer.calledMethods[0].kwargs self.assertEqual({'fieldslist': [('meta_repositoryid', 'knaw'), ('oai_id', 'record:4'), ('meta_repositorygroupid', 'knaw'), ('dare_id', 'record:4'), ('meta_collection', 'publication'), ('humanstartpage', 'http://repository.tue.nl/711504'), ('persistentid', 'URN:NBN:NL:UI:25-711504'), ('access', 'closedAccess'), ('', 'Copyright (c) Aiki, T (Toyohiko); Copyright (c) Hilhorst, D; Copyright (c) Mimura, M; Copyright (c) Muntean, A (Adrian)'), ('genre', 'article'), ('', 'Discrete and Continuous Dynamical Systems - Series S'), ('', 'Discrete and Continuous Dynamical Systems - Series S'), ('', 'Veenendaal'), ('', 'Springer'), ('', 'text'), ('fundingid', 'info:eu-repo/grantAgreement/EC/FP5/654321'), ('', 'EERA Design Tools for Offshore Wind Farm Cluster (EERA-DTOC)'), ('', 'The European Energy Research Alliance (EERA) together with some high-impact industry partners addresses the call proposing an integrated and validated design tool combining the state-of-the-art wake, yield and electrical models available in the consortium, as a plug-in architecture with possibility for third party models.'), ('', 'Funder, $ (Adrian)'), ('fundingid', 'info:eu-repo/grantAgreement/EC/FP7/282797'), ('authors', 'Aiki, T (Toyohiko)'), ('authors', 'Hilhorst, D'), ('authors', 'Mimura, M'), ('names', 'Aiki, T (Toyohiko)'), ('names', 'Hilhorst, D'), ('names', 'Mimura, M'), ('names', 'Funder, $ (Adrian)'), ('publicationid', 'info:doi/10.3934/dcdss.2012.5.1i'), ('publicationid', '1937-1632-REL'), ('pidref', 'http://repository.tue.nl/711504'), ('title_en', 'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)'), ('title', 'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)'), ('dd_year', '2014'), ('dateissued', '2014')], 'identifier': 'record:4'}, observer.calledMethods[0].kwargs)
def testLogAllPaths(self): log = CallTrace('log') writer = QueryLogWriter(log=log) writer.writeLog(defaultCollectedLogWithPath('/sru')) writer.writeLog(defaultCollectedLogWithPath('/srv')) writer.writeLog(defaultCollectedLogWithPath('/srw.php')) self.assertEquals(['log','log', 'log'], log.calledMethodNames()) self.assertEquals(['/sru', '/srv', '/srw.php'], [m.kwargs['path'] for m in log.calledMethods])
class GustosTimedLogWriterTest(SeecrTestCase): def setUp(self): super(GustosTimedLogWriterTest, self).setUp() self.observer = CallTrace() self.top = be((Observable(), (GustosTimedLogWriter(interval=0.1), (self.observer, )))) def testWriteLog(self): collectedLog = {'collected': 'log'} def fillReport(groups, collectedLog): groups['fill me'] = 'up' self.observer.methods['fillReport'] = fillReport self.top.do.writeLog(collectedLog=collectedLog) self.assertEqual(['analyseLog', 'fillReport', 'report'], self.observer.calledMethodNames()) def testWriteEmptyLog(self): collectedLog = {'collected': 'log'} self.top.do.writeLog(collectedLog=collectedLog) self.assertEqual(['analyseLog', 'fillReport'], self.observer.calledMethodNames()) def testTimedWriting(self): collectedLog = {'collected': 'log'} def fillReport(groups, collectedLog): groups['fill me'] = 'up' self.observer.methods['fillReport'] = fillReport self.top.do.writeLog(collectedLog=collectedLog) self.assertEqual(['analyseLog', 'fillReport', 'report'], self.observer.calledMethodNames()) self.top.do.writeLog(collectedLog=collectedLog) self.top.do.writeLog(collectedLog=collectedLog) self.assertEqual( ['analyseLog', 'fillReport', 'report', 'analyseLog', 'analyseLog'], self.observer.calledMethodNames()) sleep(0.11) self.top.do.writeLog(collectedLog=collectedLog) self.assertEqual([ 'analyseLog', 'fillReport', 'report', 'analyseLog', 'analyseLog', 'analyseLog', 'fillReport', 'report' ], self.observer.calledMethodNames())
def testRedirect(self): handler = BasicHttpHandler.createWithRedirect('http://example.org/here') observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest']) dna = self.build(handler, observer) response = generatorToString(dna.all.handleRequest(RequestURI="/")) self.assertEqual('HTTP/1.0 302 Found\r\nLocation: http://example.org/here\r\n\r\n', response) self.assertEqual(['handleRequest'], observer.calledMethodNames())
def testRetrieveBatch(self): request = OaiListRequest(baseurl='http://example.org/oai', verb='ListRecords', metadataPrefix='oai_dc') opener = CallTrace('urlopen', returnValues={'urlopen': StringIO(RESPONSE)}) request._urlopen = opener.urlopen batch = request.retrieveBatch() self.assertEquals(3, len(batch.items)) self.assertEquals(['urlopen'], opener.calledMethodNames()) urlopenMethod = opener.calledMethods[0] self.assertEquals((('http://example.org/oai?verb=ListRecords&metadataPrefix=oai_dc',), {}), (urlopenMethod.args, urlopenMethod.kwargs))
def testAlternativeMethod(self): handler = BasicHttpHandler(notFoundMethod=lambda path, **kwargs: 'HTTP/1.0 404\r\n\r\n%s' % path) observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest']) dna = self.build(handler, observer) response = generatorToString(dna.all.handleRequest(RequestURI="/", path='/')) self.assertEqual('HTTP/1.0 404\r\n\r\n/', response) self.assertEqual(['handleRequest'], observer.calledMethodNames())
def test404(self): handler = BasicHttpHandler() observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest']) dna = self.build(handler, observer) response = generatorToString(dna.all.handleRequest(RequestURI="/")) self.assertEqual('HTTP/1.0 404 Not Found\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<html><body>404 Not Found</body></html>', response) self.assertEqual(['handleRequest'], observer.calledMethodNames())
def testShouldUpdateDnsOnDelete(self): times = [12345678.0] registry = ServiceRegistry(stateDir=self.tempdir, reactor=CallTrace(), domainname='zp.example.org') observer = CallTrace('observer') registry.addObserver(observer) def sleep(seconds): times[0] += seconds registry._now = lambda: times[0] identifier = str(uuid4()) registry.updateService(identifier=identifier, type='api', ipAddress='127.0.0.1', infoport=1234, data={}) self.assertEquals(['updateZone'], observer.calledMethodNames()) registry.listServices() self.assertEquals(['updateZone'], observer.calledMethodNames()) sleep(2*24*60*60) services = registry.listServices() self.assertEquals(['updateZone'], observer.calledMethodNames()) self.assertTrue(identifier not in services, services)