def testIncrementalHarvestScheduleSetToNone(self):
observer = CallTrace(emptyGeneratorMethods=['add'])
oaiDownloadProcessor = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
incrementalHarvestSchedule=Schedule(period=0),
workingDirectory=self.tempdir,
xWait=False,
err=StringIO())
oaiDownloadProcessor.addObserver(observer)
consume(
oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE))))
self.assertEqual('2002-06-01T19:20:30Z', oaiDownloadProcessor._from)
self.assertNotEqual(None,
oaiDownloadProcessor._earliestNextRequestTime)
self.assertEqual(
['startOaiBatch', 'add', 'stopOaiBatch', 'signalHarvestingDone'],
observer.calledMethodNames())
observer.calledMethods.reset()
oaiDownloadProcessor.setFrom(from_=None)
oaiDownloadProcessor.setIncrementalHarvestSchedule(schedule=None)
consume(
oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE))))
self.assertEqual('2002-06-01T19:20:30Z', oaiDownloadProcessor._from)
self.assertEqual(None, oaiDownloadProcessor._earliestNextRequestTime)
self.assertEqual(
['startOaiBatch', 'add', 'stopOaiBatch', 'signalHarvestingDone'],
observer.calledMethodNames())
def testRewrite(self):
rewrite = MessageRewrite(fromMessage='this_message', toMessage='to_message')
observer = CallTrace(emptyGeneratorMethods=['to_message'])
tree = be((Observable(),
(rewrite,
(observer,),
)
))
consume(tree.all.this_message(aap='noot'))
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
consume(tree.any.this_message(aap='noot'))
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
del observer.emptyGeneratorMethods[:]
observer.calledMethods.reset()
tree.call.this_message(aap='noot')
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
tree.do.this_message(aap='noot')
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
def testSuspendTimingOut(self):
# with calltrace; happy path
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext,
timeout=3.14,
onTimeout=trace.onTimeout)
self.assertEquals([], trace.calledMethodNames())
self.assertEquals(None, suspend._timer)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'],
trace.calledMethodNames())
doNextM, addTimerM, suspendM = trace.calledMethods
self.assertEquals(((suspend, ), {}), (doNextM.args, doNextM.kwargs))
self.assertEquals(((), {
'seconds': 3.14,
'callback': suspend._timedOut
}), (addTimerM.args, addTimerM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
self.assertEquals('timerToken', suspend._timer)
trace.calledMethods.reset()
suspend._timedOut()
self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames())
onTimeoutM, whenDoneM = trace.calledMethods
self.assertEquals(((), {}), (onTimeoutM.args, onTimeoutM.kwargs))
self.assertEquals(((), {}), (whenDoneM.args, whenDoneM.kwargs))
self.assertEquals(None, suspend._timer)
self.assertRaises(TimeoutException, lambda: suspend.getResult())
def testRewrite(self):
rewrite = MessageRewrite(fromMessage='this_message',
toMessage='to_message')
observer = CallTrace(emptyGeneratorMethods=['to_message'])
tree = be((Observable(), (
rewrite,
(observer, ),
)))
consume(tree.all.this_message(aap='noot'))
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
consume(tree.any.this_message(aap='noot'))
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
del observer.emptyGeneratorMethods[:]
observer.calledMethods.reset()
tree.call.this_message(aap='noot')
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
tree.do.this_message(aap='noot')
self.assertEqual(['to_message'], observer.calledMethodNames())
self.assertEqual(dict(aap='noot'), observer.calledMethods[0].kwargs)
class DrilldownQueriesTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.dbdq = DrilldownQueries()
self.observer = CallTrace(methods=dict(executeQuery=mockExecuteQuery))
self.dbdq.addObserver(self.observer)
def testDrilldownQuery(self):
result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a = b']}))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries'])
self.observer.calledMethods.reset()
result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a exact b']}))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries'])
def testErrorForInvalidFormatDrilldownQuery(self):
try:
retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a']}))
self.fail()
except ValueError, e:
self.assertEquals('x-drilldown-query format should be field=value', str(e))
self.assertEquals([], self.observer.calledMethodNames())
def testShouldUnsetFlagImmediate(self):
i = [0]
def addTimer(*args, **kwargs):
i[0] = i[0] + 1
return i[0]
reactor = CallTrace('reactor', methods=dict(addTimer=addTimer))
registry = ServiceRegistry(reactor, self.tempdir, domainname="zp.example.org")
observer = CallTrace('observer')
registry.addObserver(observer)
identifier = str(uuid4())
registry.updateService(identifier=identifier, type='plein', ipAddress='127.0.0.1', infoport=1234, data={})
registry = ServiceRegistry(reactor, self.tempdir, domainname="zp.example.org")
flag = READABLE
service = registry.getService(identifier)
registry.setFlag(identifier, flag, True)
self.assertEqual([], reactor.calledMethodNames())
service = registry.getService(identifier)
self.assertFalse(service[flag.name], service)
state = registry.getPrivateStateFor(identifier)
self.assertTrue(state[flag.name], state)
self.assertTrue(state[flag.name + "_goingup"])
registry.setFlag(identifier, flag, False, immediate=True)
self.assertEqual([], reactor.calledMethodNames())
service = registry.getService(identifier)
self.assertFalse(service[flag.name], service)
state = registry.getPrivateStateFor(identifier)
self.assertFalse(state[flag.name], state)
self.assertFalse(flag.name + "_goingup" in state)
class RememberMeCookieTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.paths = []
def handleRequest(path, *args, **kwargs):
self.paths.append(path)
yield "RESPONSE"
def validateMethod(cookie):
return dict(
value="USER", header="COOKIE HEADER"
) if cookie == "THIS IS THE REMEMBER ME COOKIE" else None
self.observer = CallTrace(
methods={
'handleRequest': handleRequest,
'validateCookie': validateMethod,
'cookieName': lambda: "CID"
})
self.dna = be((Observable(), (RememberMeCookie(), (self.observer, ))))
def testNoCookie(self):
session = {}
response = asString(
self.dna.all.handleRequest(path='/some_page',
Headers={},
session=session))
self.assertEqual("RESPONSE", response)
self.assertEqual(['/some_page'], self.paths)
self.assertFalse('user' in session, session)
self.assertEqual(['cookieName', 'handleRequest'],
self.observer.calledMethodNames())
self.assertEquals({
'path': '/some_page',
'session': {},
'Headers': {}
}, self.observer.calledMethods[-1].kwargs)
def testCookie(self):
session = {}
Headers = dict(Cookie="CID=THIS IS THE REMEMBER ME COOKIE")
response = asString(
self.dna.all.handleRequest(path='/some_page',
Headers=Headers,
session=session))
self.assertEqual("RESPONSE", response)
self.assertEqual(['/some_page'], self.paths)
self.assertTrue('user' in session, session)
self.assertEqual(['cookieName', 'validateCookie', 'handleRequest'],
self.observer.calledMethodNames())
self.assertEquals(
{
'path': '/some_page',
'session': {
'user': '******'
},
'Headers': Headers
}, self.observer.calledMethods[-1].kwargs)
def testAddDelete(self):
observer = CallTrace('observer')
registry = ObjectRegistry(self.tempdir,
name='name',
redirectPath='/redirect')
registry.addObserver(observer)
registry.registerKeys(keys=['name', 'key2', 'another'],
booleanKeys=['enabled', 'feature'])
object1id = registry.addObject(name=["object1"],
key2=["value_1"],
enabled=['on'])
object2id = registry.addObject(name=["object2"],
key2=["value_2"],
enabled=['on'])
self.assertEquals(
{
object1id: {
'key2': 'value_1',
'enabled': True,
'name': 'object1',
'another': '',
'feature': False
},
object2id: {
'key2': 'value_2',
'enabled': True,
'name': 'object2',
'another': '',
'feature': False
}
}, registry.listObjects())
self.assertEquals(['objectAdded', 'objectAdded'],
observer.calledMethodNames())
self.assertEquals({
'name': 'name',
'identifier': object1id
}, observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
registry.removeObject(identifier=object2id)
self.assertEquals(['objectRemoved'], observer.calledMethodNames())
self.assertEquals({
'name': 'name',
'identifier': object2id
}, observer.calledMethods[0].kwargs)
registry = ObjectRegistry(self.tempdir,
name='name',
redirectPath='/redirect')
self.assertEquals(
{
object1id: {
'key2': 'value_1',
'enabled': True,
'name': 'object1',
'another': '',
'feature': False
},
}, registry.listObjects())
class OaiSetMaskTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.observer = CallTrace()
self.dna = be((Observable(), (OaiSetMask(['set1', 'set2'],
name="set1|set2 mask"),
(self.observer, ))))
def testOaiSelect(self):
self.dna.call.oaiSelect()
self.assertEqual(1, len(self.observer.calledMethods))
self.assertEqual(set(['set1', 'set2']),
self.observer.calledMethods[0].kwargs['setsMask'])
def testOaiSelectWithSetsMask(self):
self.dna.call.oaiSelect(setsMask=['set3'])
self.assertEqual(1, len(self.observer.calledMethods))
self.assertEqual(set(['set1', 'set2', 'set3']),
self.observer.calledMethods[0].kwargs['setsMask'])
def testGetUniqueInSets(self):
self.observer.returnValues['getRecord'] = MockRecord(
'id', sets=['set1', 'set2', 'set3'])
self.dna.call.getRecord('xyz')
self.assertEqual(['getRecord'], self.observer.calledMethodNames())
getRecordCall = self.observer.calledMethods[0]
self.assertEqual(('xyz', ), getRecordCall.args)
def testGetUniqueNotInSets(self):
self.observer.returnValues['getRecord'] = MockRecord('id',
sets=['set1'])
self.dna.call.getRecord('xyz')
self.assertEqual(['getRecord'], self.observer.calledMethodNames())
def testGetUniqueWithSetsMask(self):
self.observer.returnValues['getRecord'] = MockRecord(
'id', sets=['set1', 'set2', 'set3'])
self.dna.call.getRecord('xyz', setsMask=['set3'])
self.assertEqual(['getRecord'], self.observer.calledMethodNames())
getRecordCall = self.observer.calledMethods[0]
self.assertEqual(('xyz', ), getRecordCall.args)
self.observer.calledMethods.reset()
self.observer.returnValues['getRecord'] = MockRecord(
'id', sets=['set1', 'set2'])
self.dna.call.getRecord('xyz', setsMask=['set3'])
self.assertEqual(['getRecord'], self.observer.calledMethodNames())
def testOtherMethodsArePassed(self):
self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: (
x for x in [])
list(compose(self.dna.all.getAllMetadataFormats()))
self.assertEqual(1, len(self.observer.calledMethods))
self.assertEqual('getAllMetadataFormats',
self.observer.calledMethods[0].name)
class OaiPmhDownloadTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.log = StringIO()
self.oaiListRequest = CallTrace()
self.oaiListRequest.returnValues['create'] = self.oaiListRequest
self.oaiListRequest.returnValues['buildUrl'] = 'http://example.org/oai?verb=ListRequest'
self.oaiBatch = CallTrace()
self.oaiListRequest.returnValues['retrieveBatch'] = self.oaiBatch
self.oaiBatch.request = self.oaiBatch
self.oaiBatch.baseurl = 'http://base.example.org'
self.oaiBatch.resumptionToken = 'continueHere'
self.record = CallTrace()
self.record.identifier = 'identifier'
self.oaiBatch.items = [self.record]
def prepareDownload(self, repositories, **kwargs):
dl = OaiPmhDownload(repositories=repositories, log=self.log, **kwargs)
dl._OaiListRequest = self.oaiListRequest.create
return dl
def testNoRepositories(self):
dl = self.prepareDownload(repositories=[])
dl.downloadBatch({})
self.assertEquals([], self.oaiListRequest.calledMethodNames())
def testOneRepository(self):
repo1 = {
'baseurl': 'http://example.org/oai',
'metadataPrefix': 'prefix',
'repositoryGroupId': 'group'
}
dl = self.prepareDownload(repositories=[repo1])
batch = dl.downloadBatch({})
self.assertEquals(['create', 'buildUrl', 'retrieveBatch'], self.oaiListRequest.calledMethodNames())
self.assertEquals({'baseurl': 'http://example.org/oai', 'metadataPrefix': 'prefix', 'set': None}, self.oaiListRequest.calledMethods[0].kwargs)
self.assertEquals({'repositoriesRemaining': [repo1], 'resumptionToken': 'continueHere'}, batch.resumptionAttributes())
self.assertEquals(["group:identifier"], [r.identifier for r in batch.records])
self.oaiListRequest.calledMethods.reset()
batch = dl.downloadBatch(batch.resumptionAttributes())
self.assertEquals(['create', 'buildUrl', 'retrieveBatch'], self.oaiListRequest.calledMethodNames())
self.assertEquals({'baseurl': 'http://example.org/oai', 'resumptionToken':'continueHere'}, self.oaiListRequest.calledMethods[0].kwargs)
def testOneRepositoryFinishing(self):
self.oaiBatch.resumptionToken = None
repo1 = {
'baseurl': 'http://example.org/oai',
'metadataPrefix': 'prefix',
'repositoryGroupId': 'group'
}
dl = self.prepareDownload(repositories=[repo1])
batch = dl.downloadBatch({})
self.assertEquals({'repositoriesRemaining': [], 'resumptionToken': None}, batch.resumptionAttributes())
self.assertEquals('prefix', batch.metadataPrefix)
def testLogLiveExample(self):
collectedLog = {
'httpRequest': {
'timestamp': [1396596372.708574],
'Headers': [{}],
'Client': [('127.0.0.1', 57075)],
'arguments': [{
'query': ['meta.upload.id exact "NICL:oai:mdms.kenict.org:oai:nicl.nl:k163645"'],
'operation': ['searchRetrieve'],
'version': ['1.2'],
'recordPacking': ['xml'],
'recordSchema': ['smbAggregatedData']
}],
'RequestURI': ['/edurep/sruns?query=meta.upload.id+exact+%22NICL%3Aoai%3Amdms.kenict.org%3Aoai%3Anicl.nl%3Ak163645%22&operation=searchRetrieve&version=1.2&recordPacking=xml&recordSchema=smbAggregatedData'],
'query': ['query=meta.upload.id+exact+%22NICL%3Aoai%3Amdms.kenict.org%3Aoai%3Anicl.nl%3Ak163645%22&operation=searchRetrieve&version=1.2&recordPacking=xml&recordSchema=smbAggregatedData'],
'path': ['/edurep/sruns'],
'Method': ['GET'],
'HTTPVersion': ['1.0']
},
'query-scope': {
'sub-scope': {
'cqlClauses': [2],
'sru': {
'indexTime': [Decimal('0.000')],
'handlingTime': [Decimal('0.004')],
'numberOfRecords': [1],
'queryTime': [Decimal('0.003')],
'arguments': [{
'recordSchema': 'smbAggregatedData',
'version': '1.2',
'recordPacking': 'xml',
'maximumRecords': 10,
'startRecord': 1,
'query': 'meta.upload.id exact "NICL:oai:mdms.kenict.org:oai:nicl.nl:k163645"',
'operation': 'searchRetrieve'
}]
}
}
},
'httpResponse': {
'duration': [0.004216909408569336],
'httpStatus': ['200'],
'size': [1889]
}
}
log = CallTrace('log')
writer = QueryLogWriter(log=log, scopeNames=('query-scope', 'sub-scope'))
log2 = CallTrace('log')
writer2 = QueryLogWriter(log=log2, scopeNames=('query-scope', 'other-scope'))
writer.writeLog(collectedLog)
writer2.writeLog(collectedLog)
self.assertEquals(['log'], log.calledMethodNames())
self.assertEquals([], log2.calledMethodNames())
self.assertEquals(['maximumRecords=10&operation=searchRetrieve&query=meta.upload.id+exact+%22NICL%3Aoai%3Amdms.kenict.org%3Aoai%3Anicl.nl%3Ak163645%22&recordPacking=xml&recordSchema=smbAggregatedData&startRecord=1&version=1.2'], [m.kwargs['queryArguments'] for m in log.calledMethods])
def testPausePausesOnStart(self):
# autoStart
reactor = CallTrace('reactor')
pc = PeriodicCall(reactor=reactor, autoStart=False)
pc.observer_init()
self.assertEquals([], reactor.calledMethodNames())
# explicit .pause()
pc = PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True)
pc.pause()
pc.observer_init()
self.assertEquals([], reactor.calledMethodNames())
def prepare():
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames())
trace.calledMethods.reset()
suspend._timedOut()
self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames())
self.assertEquals(True, suspend._settled)
trace.calledMethods.reset()
return trace, suspend
class OaiSetMaskTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.observer = CallTrace()
self.dna = be(
(Observable(),
(OaiSetMask(['set1', 'set2'], name="set1|set2 mask"),
(self.observer,)
)
)
)
def testOaiSelect(self):
self.dna.call.oaiSelect()
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals(set(['set1', 'set2']), self.observer.calledMethods[0].kwargs['setsMask'])
def testOaiSelectWithSetsMask(self):
self.dna.call.oaiSelect(setsMask=['set3'])
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals(set(['set1', 'set2', 'set3']), self.observer.calledMethods[0].kwargs['setsMask'])
def testGetUniqueInSets(self):
self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1', 'set2', 'set3'])
self.dna.call.getRecord('xyz')
self.assertEquals(['getRecord'], self.observer.calledMethodNames())
getRecordCall = self.observer.calledMethods[0]
self.assertEquals(('xyz',), getRecordCall.args)
def testGetUniqueNotInSets(self):
self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1'])
self.dna.call.getRecord('xyz')
self.assertEquals(['getRecord'], self.observer.calledMethodNames())
def testGetUniqueWithSetsMask(self):
self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1', 'set2', 'set3'])
self.dna.call.getRecord('xyz', setsMask=['set3'])
self.assertEquals(['getRecord'], self.observer.calledMethodNames())
getRecordCall = self.observer.calledMethods[0]
self.assertEquals(('xyz',), getRecordCall.args)
self.observer.calledMethods.reset()
self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1', 'set2'])
self.dna.call.getRecord('xyz', setsMask=['set3'])
self.assertEquals(['getRecord'], self.observer.calledMethodNames())
def testOtherMethodsArePassed(self):
self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: (x for x in [])
list(compose(self.dna.all.getAllMetadataFormats()))
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals('getAllMetadataFormats', self.observer.calledMethods[0].name)
def testAdditionalArguments(self):
log = CallTrace('log')
writer = QueryLogWriter(log=log)
observer = CallTrace('additional', returnValues={'determineQueryArguments': dict(key='value')})
writer.addObserver(observer)
writer.writeLog(defaultCollectedLog())
self.assertEquals(['log'], log.calledMethodNames())
self.assertEquals(['key=value'], [m.kwargs['queryArguments'] for m in log.calledMethods])
self.assertEquals(['determineQueryArguments'], observer.calledMethodNames())
self.assertEquals(dict(
collectedLog=defaultCollectedLog(),
scopeNames=(),
currentArgs={'version': '1.2'},
), observer.calledMethods[0].kwargs)
def prepare():
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout)
self.assertEquals([], trace.calledMethodNames())
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames())
doNextM, addTimerM, suspendM = trace.calledMethods
self.assertEquals(((suspend,), {}), (doNextM.args, doNextM.kwargs))
self.assertEquals(((), {'seconds': 3.14, 'callback': suspend._timedOut}), (addTimerM.args, addTimerM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
trace.calledMethods.reset()
return trace, suspend
def testObservableCall(self):
o = Observable()
calltrace = CallTrace('calltrace')
o.addObserver(calltrace)
result = o.call.getSomething()
self.assertEqual(None, result)
self.assertEquals(['getSomething'], calltrace.calledMethodNames())
calltrace.calledMethods.reset()
calltrace.returnValues['getSomething'] = 'RESULT'
result = o.call.getSomething()
self.assertEqual('RESULT', result)
self.assertEquals(['getSomething'], calltrace.calledMethodNames())
class CombinePartsTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.combine = CombineParts({'together':['one', 'two']})
self.observer = CallTrace('observer')
self.combine.addObserver(self.observer)
self.observer.methods['getData'] = lambda identifier, name: '<%s/>' % name
def testPassThroughOtherStuff(self):
result = self.combine.getData(identifier='identifier', name='name')
self.assertEquals('<name/>', result)
self.assertEquals(['getData'], self.observer.calledMethodNames())
self.assertEquals([dict(identifier='identifier', name='name')], [m.kwargs for m in self.observer.calledMethods])
def testTogether(self):
result = self.combine.getData(identifier='identifier', name='together')
expected = '<doc:document xmlns:doc="http://meresco.org/namespace/harvester/document"><doc:part name="one"><one/></doc:part><doc:part name="two"><two/></doc:part></doc:document>'
self.assertEquals(expected, result)
self.assertEquals(['getData', 'getData'], self.observer.calledMethodNames())
self.assertEquals([dict(identifier='identifier', name='one'), dict(identifier='identifier', name='two')], [m.kwargs for m in self.observer.calledMethods])
def testTogetherWithOnePartMissingAllowed(self):
self.combine = CombineParts({'together':['one', 'two']}, allowMissingParts=['two'])
self.combine.addObserver(self.observer)
def getData(identifier, name):
if name == 'two':
raise KeyError('two')
return '<%s/>' % name
self.observer.methods['getData'] = getData
result = self.combine.getData(identifier='identifier', name='together')
self.assertEquals('<doc:document xmlns:doc="http://meresco.org/namespace/harvester/document"><doc:part name="one"><one/></doc:part></doc:document>', result)
def testTogetherWithOnePartMissingNotAllowed(self):
def getData(identifier, name):
if name == 'two':
raise KeyError('two')
return '<%s/>' % name
self.observer.methods['getData'] = getData
self.assertRaises(KeyError, lambda: self.combine.getData(identifier='identifier', name='together'))
def testTogetherWithGivenMissingPartsAllowed(self):
self.combine = CombineParts({'together':['one', 'two']}, allowMissingParts=['two'])
self.combine.addObserver(self.observer)
def getData(identifier, name):
if name == 'one':
raise KeyError('one')
return '<%s/>' % name
self.observer.methods['getData'] = getData
self.assertRaises(KeyError, lambda: self.combine.getData(identifier='identifier', name='together'))
class OaiSetSelectTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.observer = CallTrace()
with stderr_replaced() as err:
self.dna = be(
(Observable(), (OaiSetSelect(['set1',
'set2']), (self.observer, ))))
self.assertTrue(not err.getvalue() or \
'warn("OaiSetSelect is deprecated;' in err.getvalue(), err.getvalue())
def testOaiSelect(self):
self.dna.call.oaiSelect()
self.assertEqual(1, len(self.observer.calledMethods))
methodCalled = self.observer.calledMethods[0]
self.assertTrue('sets' in methodCalled.kwargs, methodCalled)
self.assertEqual(['set1', 'set2'],
self.observer.calledMethods[0].kwargs['sets'])
def testGetUniqueInSet(self):
self.observer.returnValues['getRecord'] = MockRecord('id',
sets=['set1'])
self.dna.call.getRecord('xyz')
self.assertEqual(['getRecord'], self.observer.calledMethodNames())
getRecordCall = self.observer.calledMethods[0]
self.assertEqual(('xyz', ), getRecordCall.args)
def testGetUniqueNotInSet(self):
self.observer.returnValues['getRecord'] = MockRecord('id',
sets=['set4'])
self.dna.call.getRecord('xyz')
self.assertEqual(['getRecord'], self.observer.calledMethodNames())
def testOtherMethodsArePassed(self):
self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: (
x for x in [])
list(compose(self.dna.all.getAllMetadataFormats()))
self.assertEqual(1, len(self.observer.calledMethods))
self.assertEqual('getAllMetadataFormats',
self.observer.calledMethods[0].name)
def testSetsIsNone(self):
self.dna.call.oaiSelect(sets=None)
self.assertEqual(1, len(self.observer.calledMethods))
methodCalled = self.observer.calledMethods[0]
self.assertTrue('sets' in methodCalled.kwargs, methodCalled)
self.assertEqual(['set1', 'set2'],
self.observer.calledMethods[0].kwargs['sets'])
def testAutomatic(self):
def handleRequest(**kwargs):
yield 'DATA'
observer = CallTrace(methods=dict(handleRequest=handleRequest))
vo = SruVersionAndOperation(version='8.7', recordSchema='this')
vo.addObserver(observer)
result = asString(
vo.handleRequest(arguments={
'query': ['query'],
'recordSchema': ['schema']
}))
self.assertEquals('DATA', result)
self.assertEqual(['handleRequest'], observer.calledMethodNames())
self.assertEqual(
{
'arguments': {
'query': ['query'],
'version': ['8.7'],
'operation': ['searchRetrieve'],
'recordSchema': ['schema'],
}
}, observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
result = asString(
vo.handleRequest(arguments={
'query': ['query'],
'version': ['1.2']
}))
self.assertEquals('DATA', result)
self.assertEqual(['handleRequest'], observer.calledMethodNames())
self.assertEqual(
{
'arguments': {
'query': ['query'],
'version': ['1.2'],
'operation': ['searchRetrieve'],
'recordSchema': ['this'],
}
}, observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
result = asString(vo.handleRequest(arguments={'noquery': ['noquery']}))
self.assertEqual({'arguments': {
'noquery': ['noquery'],
}}, observer.calledMethods[0].kwargs)
class OaiSetSelectTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.observer = CallTrace()
with stderr_replaced() as err:
self.dna = be(
(Observable(),
(OaiSetSelect(['set1', 'set2']),
(self.observer,)
)
)
)
self.assertTrue(not err.getvalue() or \
'warn("OaiSetSelect is deprecated;' in err.getvalue(), err.getvalue())
def testOaiSelect(self):
self.dna.call.oaiSelect()
self.assertEquals(1, len(self.observer.calledMethods))
methodCalled = self.observer.calledMethods[0]
self.assertTrue('sets' in methodCalled.kwargs, methodCalled)
self.assertEquals(['set1', 'set2'], self.observer.calledMethods[0].kwargs['sets'])
def testGetUniqueInSet(self):
self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set1'])
self.dna.call.getRecord('xyz')
self.assertEquals(['getRecord'], self.observer.calledMethodNames())
getRecordCall = self.observer.calledMethods[0]
self.assertEquals(('xyz',), getRecordCall.args)
def testGetUniqueNotInSet(self):
self.observer.returnValues['getRecord'] = MockRecord('id', sets=['set4'])
self.dna.call.getRecord('xyz')
self.assertEquals(['getRecord'], self.observer.calledMethodNames())
def testOtherMethodsArePassed(self):
self.observer.methods['getAllMetadataFormats'] = lambda *a, **kw: (x for x in [])
list(compose(self.dna.all.getAllMetadataFormats()))
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals('getAllMetadataFormats', self.observer.calledMethods[0].name)
def testSetsIsNone(self):
self.dna.call.oaiSelect(sets=None)
self.assertEquals(1, len(self.observer.calledMethods))
methodCalled = self.observer.calledMethods[0]
self.assertTrue('sets' in methodCalled.kwargs, methodCalled)
self.assertEquals(['set1', 'set2'], self.observer.calledMethods[0].kwargs['sets'])
class MultiCqlToLuceneQueryTest(TestCase):
def setUp(self):
coreAConverter = CqlToLuceneQuery([('fieldA', 1.0)],
luceneSettings=LuceneSettings())
coreBConverter = CqlToLuceneQuery([('fieldB', 1.0)],
luceneSettings=LuceneSettings())
self.converter = MultiCqlToLuceneQuery(defaultCore='A',
coreToCqlLuceneQueries=dict(
A=coreAConverter,
B=coreBConverter))
self.observer = CallTrace('Query responder',
methods={'executeQuery': executeQueryMock})
self.dna = be((Observable(), (
self.converter,
(self.observer, ),
)))
def testConvertComposedQuery(self):
q = ComposedQuery('A')
q.setCoreQuery(core='A', query=parseString('valueAQ'))
q.setCoreQuery(core='B', query=parseString('valueBQ'))
q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B',
key='keyB'))
q.addUnite(dict(core='A', query=parseString('fieldUA exact valueUA')),
dict(core='B', query=parseString('fieldUB exact valueUB')))
q.validate()
consume(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(['executeComposedQuery'],
self.observer.calledMethodNames())
self.assertEquals(repr(TermQuery(Term('fieldA', 'valueaq'))),
repr(q.queryFor('A')))
self.assertEquals(repr(TermQuery(Term('fieldB', 'valuebq'))),
repr(q.queryFor('B')))
def testResumeOrThrowOnlyOnce(self):
# A.k.a. Promise / Future like behaviour.
trace = CallTrace("Reactor")
def getSuspend():
trace.calledMethods.reset()
suspend = Suspend(doNext=trace.doNext)
self.assertEquals([], trace.calledMethodNames())
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'suspend'], trace.calledMethodNames())
doNextM, suspendM = trace.calledMethods
self.assertEquals(((suspend,), {}), (doNextM.args, doNextM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
trace.calledMethods.reset()
return suspend
suspend = getSuspend()
suspend.resume(response='whatever')
self.assertEquals(['whenDone'], trace.calledMethodNames())
trace.calledMethods.reset()
# Below no change of result, no side-effects
self.assertEquals('whatever', suspend.getResult())
try:
suspend.resume(response='DIFFERENT')
except AssertionError, e:
self.assertEquals('Suspend already settled.', str(e))
def testLoggedPathsNewStyle(self):
log = CallTrace('log')
def handleRequest(**kwargs):
yield okPlainText
yield 'result'
index = CallTrace('index', methods={'handleRequest':handleRequest})
observable = be((Observable(),
(LogCollector(),
(QueryLogWriter(log=log, scopeNames=('global', 'yesPath')),),
(LogCollectorScope('global'),
(HandleRequestLog(),
(PathFilter('/yes'),
(LogCollectorScope('yesPath'),
(index,),
)
),
(PathFilter('/no'),
(index,),
)
)
)
)
))
result = asString(observable.all.handleRequest(Client=('11.22.33.44', 1234), path='/yes'))
self.assertEquals(okPlainText+'result', result)
result = asString(observable.all.handleRequest(Client=('22.33.44.55', 2345), path='/no'))
self.assertEquals(okPlainText+'result', result)
result = asString(observable.all.handleRequest(Client=('33.44.55.66', 3456), path='/yes'))
self.assertEquals(okPlainText+'result', result)
self.assertEquals(['log', 'log'], log.calledMethodNames())
self.assertEquals(['/yes', '/yes'], [m.kwargs['path'] for m in log.calledMethods])
def testUpdateObject(self):
observer = CallTrace('observer')
registry = ObjectRegistry(self.tempdir,
name='name',
redirectPath='/redirect')
registry.addObserver(observer)
registry.registerKeys(keys=['key2', 'name'], booleanKeys=['enabled'])
object1id = registry.addObject(name=["object1"],
key2=["value_1"],
enabled=['on'])
registry.updateObject(identifier=object1id,
name=["object1"],
key2=["value_2"])
self.assertEquals(['objectAdded', 'objectUpdated'],
observer.calledMethodNames())
self.assertEquals({
'name': 'name',
'identifier': object1id
}, observer.calledMethods[-1].kwargs)
registry = ObjectRegistry(self.tempdir,
name='name',
redirectPath='/redirect')
self.assertEquals(
{
object1id: {
'key2': 'value_2',
'enabled': False,
'name': 'object1'
}
}, registry.listObjects())
class FlagCheckTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.flagCheck = FlagCheck(serviceIdentifier='identifier', flag=READABLE)
def handleRequest(**kwargs):
yield 'HTTP/1.0 200 Ok\r\nContent-Type: plain/text\r\n\r\nRESULT'
self.observer = CallTrace(emptyGeneratorMethods=['someMessage'], methods={'handleRequest':handleRequest})
self.server = be((Observable(),
(self.flagCheck,
(self.observer,),
)
))
def testAllowedWhenFlagSet(self):
consume(self.flagCheck.updateConfig(this_service={'readable': False, 'state':{'readable': True}}))
consume(self.server.all.someMessage(ignored='ignored'))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
def testNotAllowedServiceDoesntExists(self):
consume(self.flagCheck.updateConfig(this_service=None))
try:
consume(self.server.all.someMessage(ignored='ignored'))
self.fail()
except EnvironmentError, e:
self.assertEquals("'someMessage' is not allowed at the moment (readable=False).", str(e))
def testApacheLog(self):
requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"])
requestHandler.returnValues["handleRequest"] = (f for f in [Yield, okPlainText, "te", callable, "xt"])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be(
(Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),)))
)
result = asList(
observable.all.handleRequest(
Method="GET",
Client=("127.0.0.1", 1234),
RequestURI="http://example.org/path?key=value",
query="key=value",
path="/path",
Headers={"Referer": "http://meresco.org", "User-Agent": "Meresco-Components Test"},
otherKwarg="value",
)
)
self.assertEquals([Yield, okPlainText, "te", callable, "xt"], result)
self.assertEquals(["handleRequest"], requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n',
logline,
)
def testRemotePrefixSearch(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(
total=5, hits=["1", "2", "3", "4", "5"]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(
observable.any.prefixSearch(prefix='aap',
fieldname='field',
limit=10))
self.assertEquals(5, result.total)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(
{
'message': 'prefixSearch',
'kwargs': {
'prefix': 'aap',
'fieldname': 'field',
'limit': 10,
}
}, loads(m.kwargs['body']))
def testReadFromStorage(self):
inputEvent = fromstring('<document/>')
interceptor = CallTrace('Interceptor',
ignoredAttributes=[
'getData', 'all_unknown', 'any_unknown',
'call_unknown'
])
interceptor.methods['add'] = yieldNothing
storage = CallTrace('Storage',
ignoredAttributes=['add', 'all_unknown'])
storage.returnValues['getData'] = '<some>this is partone</some>'
v = createVenturiHelix(
[{
'partname': 'partone',
'xpath': '/document/part[@name="partone"]/text()'
}], [], interceptor, storage)
list(compose(v.all.add('identifier', 'document', inputEvent)))
self.assertEquals(['begin', 'add'],
[m.name for m in interceptor.calledMethods])
self.assertEquals(
'<some>this is partone</some>',
lxmltostring(interceptor.calledMethods[1].kwargs['lxmlNode']))
self.assertEquals(['begin', 'getData'], storage.calledMethodNames())
self.assertEquals(dict(identifier='identifier', name='partone'),
storage.calledMethods[1].kwargs)
def prepare():
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext,
timeout=3.14,
onTimeout=trace.onTimeout)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'],
trace.calledMethodNames())
trace.calledMethods.reset()
suspend._timedOut()
self.assertEquals(['onTimeout', 'whenDone'],
trace.calledMethodNames())
self.assertEquals(True, suspend._settled)
trace.calledMethods.reset()
return trace, suspend
def testServicePrefixSearch(self):
observer = CallTrace('lucene')
def prefixSearch(**kwargs):
raise StopIteration(LuceneResponse(total=2, hits=['aap', 'noot']))
yield
observer.methods['prefixSearch'] = prefixSearch
service = LuceneRemoteService(CallTrace('reactor'))
service.addObserver(observer)
body = dumps({
'message': 'prefixSearch',
'kwargs': {
'prefix': 'aap',
'fieldname': 'field',
'limit': 10,
}
})
result = ''.join(
compose(
service.handleRequest(path='/__lucene_remote__',
Method="POST",
Body=body)))
header, body = result.split('\r\n' * 2)
self.assertTrue('Content-Type: application/json' in header, header)
response = LuceneResponse.fromJson(body)
self.assertEquals(2, response.total)
self.assertEquals(['aap', 'noot'], response.hits)
self.assertEquals(['prefixSearch'], observer.calledMethodNames())
m = observer.calledMethods[0]
self.assertEquals('aap', m.kwargs['prefix'])
self.assertEquals(10, m.kwargs['limit'])
self.assertEquals('field', m.kwargs['fieldname'])
def testLogAsObservable(self):
log = CallTrace('log', onlySpecifiedMethods=True, methods={'log': lambda **kwargs: None})
writer = QueryLogWriter()
writer.addObserver(log)
writer.writeLog(defaultCollectedLogWithPath('/sru'))
self.assertEquals(['log'], log.calledMethodNames())
self.assertEquals(['/sru'], [m.kwargs['path'] for m in log.calledMethods])
def testResumeOrThrowOnlyOnce(self):
# A.k.a. Promise / Future like behaviour.
trace = CallTrace("Reactor")
def getSuspend():
trace.calledMethods.reset()
suspend = Suspend(doNext=trace.doNext)
self.assertEquals([], trace.calledMethodNames())
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'suspend'], trace.calledMethodNames())
doNextM, suspendM = trace.calledMethods
self.assertEquals(((suspend, ), {}),
(doNextM.args, doNextM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
trace.calledMethods.reset()
return suspend
suspend = getSuspend()
suspend.resume(response='whatever')
self.assertEquals(['whenDone'], trace.calledMethodNames())
trace.calledMethods.reset()
# Below no change of result, no side-effects
self.assertEquals('whatever', suspend.getResult())
try:
suspend.resume(response='DIFFERENT')
except AssertionError, e:
self.assertEquals('Suspend already settled.', str(e))
def testLogRequestInCaseOfExceptionAfterStatusCode(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
def handleRequestRaisesException(**kwargs):
yield Yield
yield okPlainText
yield 'text'
raise ValueError('doesntreallymatter')
yield
requestHandler.methods['handleRequest'] = handleRequestRaisesException
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(),
(LogCollector(),
(handleRequestLog,
(requestHandler,)
),
(ApacheLogWriter(stream),),
)
))
result = []
g = compose(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test'}, otherKwarg='value'))
try:
for x in g:
result.append(x)
except Exception:
pass
self.assertEqual([Yield, bytes(okPlainText, encoding='utf-8'), b'text'], result)
self.assertEqual(['handleRequest'], requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\')\n', logline)
def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' %
LuceneResponse(total=5,
hits=[
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{
'fieldname': 'field',
'maxTerms': 5
}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(
observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json',
m.kwargs['headers']['Content-Type'])
message, kwargs = jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{
'fieldname': 'field',
'maxTerms': 5
}], query.facetsFor('coreA'))
def testTcpPacketListener(self):
reactor = CallTrace('reactor')
observer = CallTrace('observer',
emptyGeneratorMethods=['handlePacket'])
tcpPacketListener = TcpPacketListener(reactor, port=1234)
server = be((Observable(), (tcpPacketListener, (observer, ))))
list(compose(server.once.observer_init()))
self.assertEquals('addReader', reactor.calledMethods[0].name)
acceptCallback = reactor.calledMethods[0].args[1]
data = "TEST" * 1024
sok = socket()
sok.connect(('localhost', 1234))
bytesSent = sok.send(data)
self.assertEquals(len(data), bytesSent)
sok.close()
acceptCallback()
self.assertEquals('addReader', reactor.calledMethods[1].name)
handleCallback = reactor.calledMethods[1].args[1]
handleCallback()
self.assertEquals('addProcess', reactor.calledMethods[-2].name)
reactor.calledMethods[-2].args[0]()
self.assertEquals(['observer_init', 'handlePacket'],
observer.calledMethodNames())
self.assertEquals(data, observer.calledMethods[1].kwargs['data'])
self.assertEquals('removeReader', reactor.calledMethods[-2].name)
self.assertEquals('removeProcess', reactor.calledMethods[-1].name)
class CountReportTest(SeecrTestCase):
def testCountOne(self):
self.report = CallTrace(returnValues={'report': None},
onlySpecifiedMethods=True)
self.top = be((Observable(), (LogCollector(), (
GustosLogWriter(),
(CountReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='calls'), ),
(CountReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='collectTimes',
keys=('collectResponse', 'some')), ),
(self.report, ),
), (CollectMeALog(), ))))
self.top.do.callMe(times=3)
self.assertEqual(['report'], self.report.calledMethodNames())
countReport = self.report.calledMethods[0].kwargs['values'][
'gustosGroup']['Counts']
self.assertEqual({
'collectTimes': {
'count': 3
},
'calls': {
'count': 1
}
}, countReport)
def testUnusedTimeoutSetInitialisesTimer(self):
# Whitebox (unusedTimeout -> addTimer)
mockReactor = CallTrace()
SocketPool(reactor=mockReactor, unusedTimeout=0.02)
self.assertEquals(['addTimer'], mockReactor.calledMethodNames())
self.assertEquals(['seconds', 'callback'], mockReactor.calledMethods[0].kwargs.keys())
self.assertEquals(0.02, mockReactor.calledMethods[0].kwargs['seconds'])
# Blackbox
def test():
top = be((Observable(),
(SocketPool(reactor=reactor(), unusedTimeout=0.02),),
))
yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('A'))
yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('B'))
yield sleep(seconds=0.001)
result = yield top.any.getPooledSocket(host='x', port=80)
self.assertEquals('B', result)
yield sleep(seconds=0.04)
result = yield top.any.getPooledSocket(host='x', port=80)
self.assertEquals(None, result)
asProcess(test())
def testAdd(self):
class Factory():
def __init__(self, observable, untokenizedFieldnames):
self.observable = observable
self.untokenizedFieldnames = untokenizedFieldnames
def fieldsFor(self, fieldname, value):
raise StopIteration([(fieldname, value)])
yield
fieldFactory = Factory
fieldRegistry = FieldRegistry(drilldownFields=[DrilldownField('drilldown.field')])
index = FieldsListToLuceneDocument(fieldRegistry, untokenizedFieldnames=[], indexFieldFactory=fieldFactory)
observer = CallTrace(emptyGeneratorMethods=['addDocument'])
index.addObserver(observer)
fields = [
("field1", "value1"),
("field2", "value2"),
("drilldown.field", "a drilldown value"),
("__key__.field", "a key value"),
("__key__.field1", 2),
]
consume(index.add(identifier="", fieldslist=fields))
self.assertEquals(['addDocument'], observer.calledMethodNames())
fields = observer.calledMethods[0].kwargs['fields']
self.assertEqual([
{'name': 'field1', 'type': 'TextField', 'value': 'value1'},
{'name': 'field2', 'type': 'TextField', 'value': 'value2'},
{'name': 'drilldown.field', 'type': 'FacetField', 'path': ['a drilldown value']},
{'name': '__key__.field', 'type': 'KeyField', 'value': 'a key value'},
{'name': '__key__.field1', 'type': 'KeyField', 'value': 2},
], fields)
def testDeleteWithError(self):
ids = Ids(self.tmp_path / "test.ids")
try:
for i in range(10):
ids.add('id:{}'.format(i))
deleteIds = DeleteIds(Bucket(id='test'))
def delete(anUpload):
if anUpload.id == 'id:7':
raise ValueError('fout')
observer = CallTrace(
methods=dict(delete=delete,
deleteIdentifier=ids.remove,
flushIds=lambda **kwargs: ids.reopen()))
deleteIds.addObserver(observer)
self.assertRaises(ValueError, lambda: deleteIds.delete(ids))
self.assertEqual(['id:7', 'id:8', 'id:9'],
readIds(join(self.tempdir, 'test.ids')))
self.assertEqual(
8,
len([m for m in observer.calledMethodNames()
if m == 'delete']))
finally:
ids.close()
def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{'fieldname': 'field', 'maxTerms':5}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json', m.kwargs['headers']['Content-Type'])
message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testUnusedTimeoutSetInitialisesTimer(self):
# Whitebox (unusedTimeout -> addTimer)
mockReactor = CallTrace()
SocketPool(reactor=mockReactor, unusedTimeout=0.02)
self.assertEquals(['addTimer'], mockReactor.calledMethodNames())
self.assertEquals(['seconds', 'callback'],
mockReactor.calledMethods[0].kwargs.keys())
self.assertEquals(0.02, mockReactor.calledMethods[0].kwargs['seconds'])
# Blackbox
def test():
top = be((
Observable(),
(SocketPool(reactor=reactor(), unusedTimeout=0.02), ),
))
yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('A'))
yield top.any.putSocketInPool(host='x', port=80, sock=MockSok('B'))
yield sleep(seconds=0.001)
result = yield top.any.getPooledSocket(host='x', port=80)
self.assertEquals('B', result)
yield sleep(seconds=0.04)
result = yield top.any.getPooledSocket(host='x', port=80)
self.assertEquals(None, result)
asProcess(test())
def testServiceExecuteQuery(self):
observer = CallTrace('lucene')
def executeQuery(**kwargs):
raise StopIteration(LuceneResponse(total=2, hits=['aap','noot']))
yield
observer.methods['executeQuery'] = executeQuery
service = LuceneRemoteService(CallTrace('reactor'))
service.addObserver(observer)
body = dumps({
'message': 'executeQuery',
'kwargs':{
'cqlAbstractSyntaxTree': {'__CQL_QUERY__': 'query AND field=value'},
'start':0,
'stop': 10,
'facets': [{'fieldname': 'field', 'maxTerms':5}],
'filterQueries': [{'__CQL_QUERY__': 'query=fiets'}],
'joinQueries': {'core1': {'__CQL_QUERY__': 'query=test'}}
}
})
result = ''.join(compose(service.handleRequest(path='/__lucene_remote__', Method="POST", Body=body)))
header, body = result.split('\r\n'*2)
self.assertTrue('Content-Type: application/json' in header, header+body)
response = LuceneResponse.fromJson(body)
self.assertEquals(2, response.total)
self.assertEquals(['aap', 'noot'], response.hits)
self.assertEquals(['executeQuery'], observer.calledMethodNames())
m = observer.calledMethods[0]
self.assertEquals(parseString('query AND field=value'), m.kwargs['cqlAbstractSyntaxTree'])
self.assertEquals(0, m.kwargs['start'])
self.assertEquals(10, m.kwargs['stop'])
self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], m.kwargs['facets'])
self.assertEquals([parseString('query=fiets')], m.kwargs['filterQueries'])
self.assertEquals({'core1': parseString('query=test')}, m.kwargs['joinQueries'])
def testServicePrefixSearch(self):
observer = CallTrace('lucene')
def prefixSearch(**kwargs):
raise StopIteration(LuceneResponse(total=2, hits=['aap','noot']))
yield
observer.methods['prefixSearch'] = prefixSearch
service = LuceneRemoteService(CallTrace('reactor'))
service.addObserver(observer)
body = dumps({
'message': 'prefixSearch',
'kwargs':{
'prefix':'aap',
'fieldname': 'field',
'limit': 10,
}
})
result = ''.join(compose(service.handleRequest(path='/__lucene_remote__', Method="POST", Body=body)))
header, body = result.split('\r\n'*2)
self.assertTrue('Content-Type: application/json' in header, header)
response = LuceneResponse.fromJson(body)
self.assertEquals(2, response.total)
self.assertEquals(['aap', 'noot'], response.hits)
self.assertEquals(['prefixSearch'], observer.calledMethodNames())
m = observer.calledMethods[0]
self.assertEquals('aap', m.kwargs['prefix'])
self.assertEquals(10, m.kwargs['limit'])
self.assertEquals('field', m.kwargs['fieldname'])
class ResponseTimeReportTest(SeecrTestCase):
def testCountOne(self):
self.report = CallTrace(returnValues={'report': {
"packet": "data"
}},
onlySpecifiedMethods=True)
self.top = be((Observable(), (LogCollector(), (
GustosLogWriter(),
(ResponseTimeReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='http'), ),
(ResponseTimeReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='other',
keys=('otherResponse', 'duration')), ),
(self.report, ),
), (CollectMeALog(), ))))
self.top.do.callMe(times=3)
self.assertEqual(['report'], self.report.calledMethodNames())
theReport = self.report.calledMethods[0].kwargs['values'][
'gustosGroup']['ResponseTime']
self.assertEqual({
'other': {
'time': 0.3
},
'http': {
'time': 0.0
}
}, theReport)
def testTcpPacketListener(self):
reactor = CallTrace('reactor')
observer = CallTrace('observer', emptyGeneratorMethods=['handlePacket'])
tcpPacketListener = TcpPacketListener(reactor, port=1234)
server = be((Observable(),
(tcpPacketListener,
(observer,)
)
))
list(compose(server.once.observer_init()))
self.assertEquals('addReader', reactor.calledMethods[0].name)
acceptCallback = reactor.calledMethods[0].args[1]
data = "TEST" * 1024
sok = socket()
sok.connect(('localhost', 1234))
bytesSent = sok.send(data)
self.assertEquals(len(data), bytesSent)
sok.close()
acceptCallback()
self.assertEquals('addReader', reactor.calledMethods[1].name)
handleCallback = reactor.calledMethods[1].args[1]
handleCallback()
self.assertEquals('addProcess', reactor.calledMethods[-2].name)
reactor.calledMethods[-2].args[0]()
self.assertEquals(['observer_init', 'handlePacket'], observer.calledMethodNames())
self.assertEquals(data, observer.calledMethods[1].kwargs['data'])
self.assertEquals('removeReader', reactor.calledMethods[-2].name)
self.assertEquals('removeProcess', reactor.calledMethods[-1].name)
def testApacheLog(self):
requestHandler = CallTrace(
'handler', ignoredAttributes=['writeLog', 'do_unknown'])
requestHandler.returnValues['handleRequest'] = (
f for f in [Yield, okPlainText, 'te', callable, 'xt'])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(), (
LogCollector(),
(handleRequestLog, (requestHandler, )),
(ApacheLogWriter(stream), ),
)))
result = asList(
observable.all.handleRequest(
Method='GET',
Client=('127.0.0.1', 1234),
RequestURI='http://example.org/path?key=value',
query='key=value',
path='/path',
Headers={
'Referer': 'http://meresco.org',
'User-Agent': 'Meresco-Components Test'
},
otherKwarg='value'))
self.assertEquals([Yield, okPlainText, 'te', callable, 'xt'], result)
self.assertEquals(['handleRequest'],
requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n',
logline)
def testGetRecordWithRepositoryIdentifier(self):
oaigetrecord = OaiGetRecord(OaiRepository(identifier='example.org'))
record = CallTrace('record')
record.identifier = 'id0'
record.prefixes = ['oai_dc']
record.sets = []
record.isDeleted = False
observer = CallTrace(
returnValues={
'isKnownPrefix': True,
'getRecord': record
},
emptyGeneratorMethods=['oaiWatermark', 'oaiRecord'])
oaigetrecord.addObserver(observer)
consume(
oaigetrecord.getRecord(arguments=dict(
verb=['GetRecord'],
metadataPrefix=['oai_dc'],
identifier=['oai:example.org:id0'],
),
**self.httpkwargs))
self.assertEqual(
['getRecord', 'isKnownPrefix', 'oaiWatermark', 'oaiRecord'],
observer.calledMethodNames())
self.assertEqual(dict(identifier='id0', metadataPrefix='oai_dc'),
observer.calledMethods[0].kwargs)
class DrilldownQueriesTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.dbdq = DrilldownQueries()
self.observer = CallTrace(methods=dict(executeQuery=mockExecuteQuery))
self.dbdq.addObserver(self.observer)
def testDrilldownQuery(self):
result = retval(
self.dbdq.executeQuery(
extraArguments={'x-drilldown-query': ['a = b']}))
self.assertEqual('result', result)
self.assertEqual(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual([('a', ['b'])],
executeQueryMethod.kwargs['drilldownQueries'])
self.observer.calledMethods.reset()
result = retval(
self.dbdq.executeQuery(
extraArguments={'x-drilldown-query': ['a exact b']}))
self.assertEqual('result', result)
self.assertEqual(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual([('a', ['b'])],
executeQueryMethod.kwargs['drilldownQueries'])
def testErrorForInvalidFormatDrilldownQuery(self):
try:
retval(
self.dbdq.executeQuery(
extraArguments={'x-drilldown-query': ['a']}))
self.fail()
except ValueError as e:
self.assertEqual('x-drilldown-query format should be field=value',
str(e))
self.assertEqual([], self.observer.calledMethodNames())
def testNoDrilldownQuery(self):
result = retval(self.dbdq.executeQuery(extraArguments={}, query='*'))
self.assertEqual('result', result)
self.assertEqual(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual([], executeQueryMethod.kwargs['drilldownQueries'])
self.assertEqual("*", executeQueryMethod.kwargs['query'])
def testLogForNumberOfRecordsSelection(self):
log = CallTrace('log')
writer = QueryLogWriter(log=log, numberOfRecordsSelection=dict(scope='myscope', key='total'))
collectedLog = defaultCollectedLog()
collectedLog['myscope'] = {'total': [100]}
writer.writeLog(collectedLog)
self.assertEquals(['log'], log.calledMethodNames())
self.assertEquals([100], [m.kwargs['numberOfRecords'] for m in log.calledMethods])
def testLogForArgumentsInsteadOfSruArguments(self):
log = CallTrace('log')
writer = QueryLogWriter.forHttpArguments(log=log)
collectedLog = defaultCollectedLog()
collectedLog['httpRequest']['arguments'] = [{'verb':'ListRecords', 'metadataPrefix':'rdf'}]
writer.writeLog(collectedLog)
self.assertEquals(['log'], log.calledMethodNames())
self.assertEquals(['metadataPrefix=rdf&verb=ListRecords'], [m.kwargs['queryArguments'] for m in log.calledMethods])
def testOne(self):
normdocToFieldsList = NormdocToFieldsList()
observer = CallTrace(emptyGeneratorMethods=['add'])
normdocToFieldsList.addObserver(observer)
consume(normdocToFieldsList.add(identifier='record:4', lxmlNode=parse(open("data/normdoc.xml")) ))
self.assertEqual(['add'], observer.calledMethodNames())
# print "FIELDLIST:", observer.calledMethods[0].kwargs
self.assertEqual({'fieldslist': [('meta_repositoryid', 'knaw'), ('oai_id', 'record:4'), ('meta_repositorygroupid', 'knaw'), ('dare_id', 'record:4'), ('meta_collection', 'publication'), ('humanstartpage', 'http://repository.tue.nl/711504'), ('persistentid', 'URN:NBN:NL:UI:25-711504'), ('access', 'closedAccess'), ('', 'Copyright (c) Aiki, T (Toyohiko); Copyright (c) Hilhorst, D; Copyright (c) Mimura, M; Copyright (c) Muntean, A (Adrian)'), ('genre', 'article'), ('', 'Discrete and Continuous Dynamical Systems - Series S'), ('', 'Discrete and Continuous Dynamical Systems - Series S'), ('', 'Veenendaal'), ('', 'Springer'), ('', 'text'), ('fundingid', 'info:eu-repo/grantAgreement/EC/FP5/654321'), ('', 'EERA Design Tools for Offshore Wind Farm Cluster (EERA-DTOC)'), ('', 'The European Energy Research Alliance (EERA) together with some high-impact industry partners addresses the call proposing an integrated and validated design tool combining the state-of-the-art wake, yield and electrical models available in the consortium, as a plug-in architecture with possibility for third party models.'), ('', 'Funder, $ (Adrian)'), ('fundingid', 'info:eu-repo/grantAgreement/EC/FP7/282797'), ('authors', 'Aiki, T (Toyohiko)'), ('authors', 'Hilhorst, D'), ('authors', 'Mimura, M'), ('names', 'Aiki, T (Toyohiko)'), ('names', 'Hilhorst, D'), ('names', 'Mimura, M'), ('names', 'Funder, $ (Adrian)'), ('publicationid', 'info:doi/10.3934/dcdss.2012.5.1i'), ('publicationid', '1937-1632-REL'), ('pidref', 'http://repository.tue.nl/711504'), ('title_en', 'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)'), ('title', 'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)'), ('dd_year', '2014'), ('dateissued', '2014')], 'identifier': 'record:4'}, observer.calledMethods[0].kwargs)
def testLogAllPaths(self):
log = CallTrace('log')
writer = QueryLogWriter(log=log)
writer.writeLog(defaultCollectedLogWithPath('/sru'))
writer.writeLog(defaultCollectedLogWithPath('/srv'))
writer.writeLog(defaultCollectedLogWithPath('/srw.php'))
self.assertEquals(['log','log', 'log'], log.calledMethodNames())
self.assertEquals(['/sru', '/srv', '/srw.php'], [m.kwargs['path'] for m in log.calledMethods])
class GustosTimedLogWriterTest(SeecrTestCase):
def setUp(self):
super(GustosTimedLogWriterTest, self).setUp()
self.observer = CallTrace()
self.top = be((Observable(), (GustosTimedLogWriter(interval=0.1),
(self.observer, ))))
def testWriteLog(self):
collectedLog = {'collected': 'log'}
def fillReport(groups, collectedLog):
groups['fill me'] = 'up'
self.observer.methods['fillReport'] = fillReport
self.top.do.writeLog(collectedLog=collectedLog)
self.assertEqual(['analyseLog', 'fillReport', 'report'],
self.observer.calledMethodNames())
def testWriteEmptyLog(self):
collectedLog = {'collected': 'log'}
self.top.do.writeLog(collectedLog=collectedLog)
self.assertEqual(['analyseLog', 'fillReport'],
self.observer.calledMethodNames())
def testTimedWriting(self):
collectedLog = {'collected': 'log'}
def fillReport(groups, collectedLog):
groups['fill me'] = 'up'
self.observer.methods['fillReport'] = fillReport
self.top.do.writeLog(collectedLog=collectedLog)
self.assertEqual(['analyseLog', 'fillReport', 'report'],
self.observer.calledMethodNames())
self.top.do.writeLog(collectedLog=collectedLog)
self.top.do.writeLog(collectedLog=collectedLog)
self.assertEqual(
['analyseLog', 'fillReport', 'report', 'analyseLog', 'analyseLog'],
self.observer.calledMethodNames())
sleep(0.11)
self.top.do.writeLog(collectedLog=collectedLog)
self.assertEqual([
'analyseLog', 'fillReport', 'report', 'analyseLog', 'analyseLog',
'analyseLog', 'fillReport', 'report'
], self.observer.calledMethodNames())
def testRedirect(self):
handler = BasicHttpHandler.createWithRedirect('http://example.org/here')
observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest'])
dna = self.build(handler, observer)
response = generatorToString(dna.all.handleRequest(RequestURI="/"))
self.assertEqual('HTTP/1.0 302 Found\r\nLocation: http://example.org/here\r\n\r\n', response)
self.assertEqual(['handleRequest'], observer.calledMethodNames())
def testRetrieveBatch(self):
request = OaiListRequest(baseurl='http://example.org/oai', verb='ListRecords', metadataPrefix='oai_dc')
opener = CallTrace('urlopen', returnValues={'urlopen': StringIO(RESPONSE)})
request._urlopen = opener.urlopen
batch = request.retrieveBatch()
self.assertEquals(3, len(batch.items))
self.assertEquals(['urlopen'], opener.calledMethodNames())
urlopenMethod = opener.calledMethods[0]
self.assertEquals((('http://example.org/oai?verb=ListRecords&metadataPrefix=oai_dc',), {}), (urlopenMethod.args, urlopenMethod.kwargs))
def testAlternativeMethod(self):
handler = BasicHttpHandler(notFoundMethod=lambda path, **kwargs: 'HTTP/1.0 404\r\n\r\n%s' % path)
observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest'])
dna = self.build(handler, observer)
response = generatorToString(dna.all.handleRequest(RequestURI="/", path='/'))
self.assertEqual('HTTP/1.0 404\r\n\r\n/', response)
self.assertEqual(['handleRequest'], observer.calledMethodNames())
def test404(self):
handler = BasicHttpHandler()
observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest'])
dna = self.build(handler, observer)
response = generatorToString(dna.all.handleRequest(RequestURI="/"))
self.assertEqual('HTTP/1.0 404 Not Found\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<html><body>404 Not Found</body></html>', response)
self.assertEqual(['handleRequest'], observer.calledMethodNames())
def testShouldUpdateDnsOnDelete(self):
times = [12345678.0]
registry = ServiceRegistry(stateDir=self.tempdir, reactor=CallTrace(), domainname='zp.example.org')
observer = CallTrace('observer')
registry.addObserver(observer)
def sleep(seconds):
times[0] += seconds
registry._now = lambda: times[0]
identifier = str(uuid4())
registry.updateService(identifier=identifier, type='api', ipAddress='127.0.0.1', infoport=1234, data={})
self.assertEquals(['updateZone'], observer.calledMethodNames())
registry.listServices()
self.assertEquals(['updateZone'], observer.calledMethodNames())
sleep(2*24*60*60)
services = registry.listServices()
self.assertEquals(['updateZone'], observer.calledMethodNames())
self.assertTrue(identifier not in services, services)
