def testAutoStartOrScheduleRequired(self):
reactor = CallTrace('reactor')
self.assertRaises(ValueError, lambda: PeriodicCall(reactor=reactor))
self.assertRaises(
ValueError, lambda: PeriodicCall(reactor=reactor, autoStart=True))
try:
PeriodicCall(reactor=reactor, autoStart=False)
except:
self.fail('Unexpected exception')
try:
PeriodicCall(reactor=reactor,
schedule=Schedule(period=1),
autoStart=False)
except:
self.fail('Unexpected exception')
try:
PeriodicCall(reactor=reactor,
schedule=Schedule(period=1),
autoStart=True)
except:
self.fail('Unexpected exception')
def newDNA(self, **kwargs):
emptyGeneratorMethods = [kwargs.get('message', 'handle')]
self.reactor = CallTrace('Reactor', returnValues={'addTimer': 'TOKEN'})
self.observer = CallTrace('Observer', emptyGeneratorMethods=emptyGeneratorMethods, ignoredAttributes=['observer_init'])
self.pc = PeriodicCall(self.reactor, **kwargs)
self.dna = be((Observable(),
(self.pc,
(self.observer,),
),
))
def testPausePausesOnStart(self):
# autoStart
reactor = CallTrace('reactor')
pc = PeriodicCall(reactor=reactor, autoStart=False)
pc.observer_init()
self.assertEquals([], reactor.calledMethodNames())
# explicit .pause()
pc = PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True)
pc.pause()
pc.observer_init()
self.assertEquals([], reactor.calledMethodNames())
def jvmMonitorTree(reactor, updatableGustosClient):
gustosInterval = 1 if __builtins__.get('__test__', False) else 60
return be((
PeriodicCall(reactor=reactor,
schedule=Schedule(period=gustosInterval)),
(
JvmMonitor(),
(updatableGustosClient, ),
),
))
class PeriodicCallTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.newDNA(schedule=Schedule(period=3600), errorSchedule=Schedule(period=15), prio=9, name='obs_name')
list(compose(self.dna.once.observer_init()))
def newDNA(self, **kwargs):
emptyGeneratorMethods = [kwargs.get('message', 'handle')]
self.reactor = CallTrace('Reactor', returnValues={'addTimer': 'TOKEN'})
self.observer = CallTrace('Observer', emptyGeneratorMethods=emptyGeneratorMethods, ignoredAttributes=['observer_init'])
self.pc = PeriodicCall(self.reactor, **kwargs)
self.dna = be((Observable(),
(self.pc,
(self.observer,),
),
))
def testWithoutData(self):
self.assertEquals('obs_name', self.pc.observable_name())
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs))
# addTimer(3600, pc._periodicCall)
self.reactor.calledMethods.reset()
addTimer.args[1]()
self.assertEquals(['addProcess'], self.reactor.calledMethodNames())
addProcess, = self.reactor.calledMethods
self.assertEquals((1, 1), (len(addProcess.args), len(addProcess.kwargs)))
self.assertEquals({'prio': 9}, addProcess.kwargs)
thisNext = addProcess.args[0]
self.reactor.calledMethods.reset()
self.assertEquals([], self.observer.calledMethodNames())
thisNext()
self.assertEquals(['handle'], self.observer.calledMethodNames())
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
removeProcess, addTimer = self.reactor.calledMethods
self.assertEquals(((), {}), (removeProcess.args, removeProcess.kwargs))
self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs))
def testWithoutDataWithDefaultValues(self):
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs))
self.reactor.calledMethods.reset()
addTimer.args[1]()
self.assertEquals(['addProcess'], self.reactor.calledMethodNames())
addProcess, = self.reactor.calledMethods
self.assertEquals((1, 1), (len(addProcess.args), len(addProcess.kwargs)))
self.assertEquals({'prio': 9}, addProcess.kwargs)
thisNext = addProcess.args[0]
self.reactor.calledMethods.reset()
self.assertEquals([], self.observer.calledMethodNames())
thisNext()
self.assertEquals(['handle'], self.observer.calledMethodNames())
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
removeProcess, addTimer = self.reactor.calledMethods
self.assertEquals(((), {}), (removeProcess.args, removeProcess.kwargs))
self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs))
def testAutoStartOrScheduleRequired(self):
reactor = CallTrace('reactor')
self.assertRaises(ValueError, lambda: PeriodicCall(reactor=reactor))
self.assertRaises(ValueError, lambda: PeriodicCall(reactor=reactor, autoStart=True))
try: PeriodicCall(reactor=reactor, autoStart=False)
except: self.fail('Unexpected exception')
try: PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=False)
except: self.fail('Unexpected exception')
try: PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True)
except: self.fail('Unexpected exception')
def testInitialSchedule(self):
self.newDNA(initialSchedule=Schedule(period=0), schedule=Schedule(period=12))
list(compose(self.dna.once.observer_init()))
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
self.assertEquals(0, addTimer.args[0])
addTimer.args[1]()
addProcess, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addProcess.args[0]()
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
removeProcess, addTimer = self.reactor.calledMethods
self.assertEquals(12, addTimer.args[0])
def testSetSchedule(self):
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addTimer.args[1]()
self.pc.setSchedule(schedule=Schedule(period=1))
addProcess, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addProcess.args[0]()
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
removeProcess, addTimer = self.reactor.calledMethods
self.assertEquals(1, addTimer.args[0])
def testSetScheduleWithIdenticalScheduleDoesNothing(self):
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
self.assertEquals(3600, addTimer.args[0])
self.pc.setSchedule(schedule=Schedule(period=3600))
self.assertEquals([], self.reactor.calledMethodNames())
def testSetScheduleAddsANewTimer(self):
self.reactor.calledMethods.reset()
self.pc.setSchedule(schedule=Schedule(period=123))
self.assertEquals(['removeTimer', 'addTimer'], self.reactor.calledMethodNames())
removeTimer, addTimer = self.reactor.calledMethods
self.assertEquals('TOKEN', removeTimer.args[0])
self.assertEquals(123, addTimer.args[0])
def testErrorIntervalAndLoggedMessage(self):
def raiser():
raise Exception('exception')
yield
self.observer.methods['handle'] = raiser
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addTimer.args[1]()
addProcess, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
with stderr_replaced() as err:
addProcess.args[0]()
errValue = err.getvalue()
self.assertTrue(errValue.startswith(repr(self.pc)))
self.assertTrue('Traceback' in errValue, errValue)
self.assertTrue('Exception: exception' in errValue, errValue)
self.assertEquals('exception', self.pc.getState().errorState)
self.assertEquals(['handle'], self.observer.calledMethodNames())
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
_, addTimer = self.reactor.calledMethods
self.assertEquals(((15, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs))
def testErrorStateReset(self):
handleCalledBefore = []
def handle():
if not handleCalledBefore:
handleCalledBefore.append(True)
raise Exception('exception')
yield Yield
self.observer.methods['handle'] = handle
addTimer, = self.reactor.calledMethods
timerCallback = addTimer.args[1]
self.reactor.calledMethods.reset()
timerCallback()
addProcess, = self.reactor.calledMethods
addProcessCallback = addProcess.args[0]
self.reactor.calledMethods.reset()
with stderr_replaced():
addProcessCallback()
self.assertEquals('exception', self.pc.getState().errorState)
self.assertEquals(['handle'], self.observer.calledMethodNames())
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
_, addTimer = self.reactor.calledMethods
timerCallback = addTimer.args[1]
self.reactor.calledMethods.reset()
timerCallback()
addProcess, = self.reactor.calledMethods
addProcessCallback = addProcess.args[0]
self.reactor.calledMethods.reset()
addProcessCallback()
self.assertEquals(['handle', 'handle'], self.observer.calledMethodNames())
addProcessCallback()
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
self.assertEquals(None, self.pc.getState().errorState)
def testFatalErrorReRaised(self):
for exception in [KeyboardInterrupt, SystemExit, AssertionError]:
self.newDNA(schedule=Schedule(period=987))
def raiser():
raise exception('msg')
yield
self.observer.methods['handle'] = raiser
list(compose(self.dna.once.observer_init()))
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addTimer.args[1]()
addProcess, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
try:
addProcess.args[0]()
except exception:
pass
else:
self.fail()
self.assertEquals(['handle'], self.observer.calledMethodNames())
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
def testHandleWithCallableAndData(self):
suspend = CallTrace('Suspend', returnValues={'__call__': lambda *a, **kw: None})
handleLog = []
def handle():
handleLog.append('ignored-da')
yield 'ignored-da'
handleLog.append('Suspend')
yield suspend
handleLog.append('ignored-ta')
yield 'ignored-ta'
handleLog.append('Stop')
self.observer.methods['handle'] = handle
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addTimer.args[1]()
addProcess, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
# 'ignored-da'
self.assertEquals([], handleLog)
addProcess.args[0]()
self.assertEquals(['handle'], self.observer.calledMethodNames())
handleCall, = self.observer.calledMethods
self.assertEquals(((), {}), (handleCall.args, handleCall.kwargs))
self.assertEquals(['ignored-da'], handleLog)
self.assertEquals([], self.reactor.calledMethodNames())
self.observer.calledMethods.reset()
self.reactor.calledMethods.reset()
# suspend (suspend(reactor, this.next))
self.assertEquals([], suspend.calledMethodNames())
addProcess.args[0]()
self.assertEquals(['ignored-da', 'Suspend'], handleLog)
self.assertEquals([], self.observer.calledMethodNames())
self.assertEquals([], self.reactor.calledMethodNames())
self.assertEquals(['__call__'], suspend.calledMethodNames())
dunderCall, = suspend.calledMethods
self.assertEquals(2, len(dunderCall.args))
self.assertEquals(self.reactor, dunderCall.args[0])
self.assertEquals(GeneratorType, type(dunderCall.args[1].__self__))
self.assertEquals('_periodicCall', dunderCall.args[1].__self__.gi_frame.f_code.co_name)
self.assertEquals({}, dunderCall.kwargs)
self.observer.calledMethods.reset()
self.reactor.calledMethods.reset()
suspend.calledMethods.reset()
# suspend (suspend.resumeProcess())
dunderCall.args[1]()
self.assertEquals(['resumeProcess'], suspend.calledMethodNames())
resumeProcess, = suspend.calledMethods
self.assertEquals(((), {}), (resumeProcess.args, resumeProcess.kwargs))
self.assertEquals([], self.reactor.calledMethodNames())
self.assertEquals([], self.observer.calledMethodNames())
self.observer.calledMethods.reset()
self.reactor.calledMethods.reset()
suspend.calledMethods.reset()
# 'ignored-ta'
addProcess.args[0]()
self.assertEquals(['ignored-da', 'Suspend', 'ignored-ta'], handleLog)
self.assertEquals([], suspend.calledMethodNames())
self.assertEquals([], self.reactor.calledMethodNames())
self.assertEquals([], self.observer.calledMethodNames())
# 'Stop'
addProcess.args[0]()
self.assertEquals(['ignored-da', 'Suspend', 'ignored-ta', 'Stop'], handleLog)
self.assertEquals([], suspend.calledMethodNames())
self.assertEquals([], self.observer.calledMethodNames())
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
removeProcess, addTimer = self.reactor.calledMethods
self.assertEquals(((), {}), (removeProcess.args, removeProcess.kwargs))
self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs))
@stderr_replaced
def testPausePausesOnStart(self):
# autoStart
reactor = CallTrace('reactor')
pc = PeriodicCall(reactor=reactor, autoStart=False)
pc.observer_init()
self.assertEquals([], reactor.calledMethodNames())
# explicit .pause()
pc = PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True)
pc.pause()
pc.observer_init()
self.assertEquals([], reactor.calledMethodNames())
@stderr_replaced
def testPausePausesWhenRunning(self):
self.newDNA(schedule=Schedule(period=1), autoStart=True)
list(compose(self.dna.once.observer_init()))
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
# pauses after completing current task
self.pc.pause()
self.reactor.calledMethods.reset()
addTimer.args[1]()
self.assertEquals(['addProcess'], self.reactor.calledMethodNames())
addProcess, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addProcess.args[0]()
self.assertEquals(['handle'], self.observer.calledMethodNames())
self.assertEquals(['removeProcess'], self.reactor.calledMethodNames())
def testPauseRemovesInitialPendingTimer(self):
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
self.reactor.calledMethods.reset()
with stderr_replaced() as err:
self.pc.pause()
self.assertEquals('%s: paused\n' % repr(self.pc), err.getvalue())
self.assertEquals(['removeTimer'], self.reactor.calledMethodNames())
removeTimer, = self.reactor.calledMethods
self.assertEquals((('TOKEN',), {}), (removeTimer.args, removeTimer.kwargs))
def testPauseRemovesEndOfProcessPendingTimer(self):
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
# Process until timer added again
addTimer.args[1]()
addProcess, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
addProcess.args[0]()
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
self.reactor.calledMethods.reset()
# pause at pending timer
with stderr_replaced() as err:
self.pc.pause()
self.assertEquals('%s: paused\n' % repr(self.pc), err.getvalue())
self.assertEquals(['removeTimer'], self.reactor.calledMethodNames())
removeTimer, = self.reactor.calledMethods
self.assertEquals((('TOKEN',), {}), (removeTimer.args, removeTimer.kwargs))
def testResumeStartsWhenPaused(self):
self.newDNA(schedule=Schedule(period=2), autoStart=False)
list(compose(self.dna.once.observer_init()))
self.assertEquals([], self.reactor.calledMethodNames())
with stderr_replaced() as err:
self.pc.resume()
self.assertEquals('%s: resumed\n' % repr(self.pc), err.getvalue())
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
self.assertEquals(self.pc._periodicCall, addTimer.args[1])
# finish one task (1/2)
self.reactor.calledMethods.reset()
addTimer.args[1]()
addProcess, = self.reactor.calledMethods
# finish one task (2/2)
self.reactor.calledMethods.reset()
addProcess.args[0]()
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
@stderr_replaced
def testDoNotResumeNotPaused(self):
self.newDNA(schedule=Schedule(period=1), autoStart=True)
self.pc.resume()
self.assertEquals([], self.reactor.calledMethodNames())
list(compose(self.dna.once.observer_init()))
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
self.reactor.calledMethods.reset()
self.pc.resume()
self.assertEquals([], self.reactor.calledMethodNames())
addTimer.args[1]()
self.assertEquals(['addProcess'], self.reactor.calledMethodNames())
self.reactor.calledMethods.reset()
self.pc.resume()
self.assertEquals([], self.reactor.calledMethodNames())
@stderr_replaced
def testResumeOnlyAddsATimerWhenNotBusy(self):
# Because at the end of the busy / reactor processing it
# will add a timer when not paused.
addTimer, = self.reactor.calledMethods
# "Firing" timer callback
self.reactor.calledMethods.reset()
addTimer.args[1]()
self.assertEquals(['addProcess'], self.reactor.calledMethodNames())
addProcess, = self.reactor.calledMethods
busybusy = []
def handleBusyMock():
yield busybusy.append(True)
yield busybusy.append(True)
self.observer.methods['handle'] = handleBusyMock
# Busy processing, so pause won't work immediately
self.reactor.calledMethods.reset()
self.pc.pause() # <-- pause
self.assertEquals([], self.reactor.calledMethodNames())
self.assertEquals([], busybusy)
addProcess.args[0]() # <-- doing stuff
self.assertEquals([], self.reactor.calledMethodNames())
self.assertEquals([True], busybusy)
self.pc.resume() # <-- resume called
addProcess.args[0]() # <-- doing more stuff
self.assertEquals([True, True], busybusy)
self.assertEquals([], self.reactor.calledMethodNames())
addProcess.args[0]() # <-- done doing and delayed resume (addTimer)
self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
@stderr_replaced
def testGetState(self):
state = self.pc.getState()
self.assertEquals('obs_name', state.name)
self.assertEquals(False, state.paused)
self.assertEquals(Schedule(period=3600), state.schedule)
self.assertEquals({'paused': False, 'name': 'obs_name', 'schedule': Schedule(period=3600)}, state.asDict())
self.pc.observable_setName('dashy')
self.assertEquals('dashy', state.name)
self.assertEquals(None, state.errorState)
self.pc.setSchedule(schedule=Schedule(period=5))
self.assertEquals(Schedule(period=5), state.schedule)
self.pc.pause()
self.assertEquals(True, state.paused)
self.pc.resume()
addTimer = self.reactor.calledMethods[-1]
self.reactor.calledMethods.reset()
self.assertEquals(False, state.paused)
addTimer.args[1]()
self.pc.pause()
self.assertEquals(False, state.paused)
addProcess = self.reactor.calledMethods[-1]
addProcess.args[0]()
self.assertEquals(True, state.paused)
def testRepr(self):
self.assertEquals("PeriodicCall(name='obs_name', paused=False, schedule=Schedule(period=3600))", repr(self.pc))
def testShorten(self):
message = 'x' * (MAX_LENGTH)
self.assertEquals(message, message)
message = 'x' * (MAX_LENGTH) + 'z'
self.assertNotEqual(message, shorten(message))
self.assertEquals(MAX_LENGTH + len(' ... '), len(shorten(message)))
self.assertTrue(' ... ' in shorten(message), shorten(message))
self.assertTrue(shorten(message).startswith('xxxx'), shorten(message))
self.assertTrue(shorten(message).endswith('xxxz'), shorten(message))
def testMessageConfigurable(self):
self.newDNA(message="aMessage", schedule=Schedule(period=3600), errorSchedule=Schedule(period=15), prio=9, name='obs_name')
list(compose(self.dna.once.observer_init()))
self.assertEquals(['addTimer'], self.reactor.calledMethodNames())
addTimer, = self.reactor.calledMethods
callback = addTimer.args[1]
self.reactor.calledMethods.reset()
callback()
self.assertEquals(['addProcess'], self.reactor.calledMethodNames())
addProcess, = self.reactor.calledMethods
callback = addProcess.args[0]
self.reactor.calledMethods.reset()
callback()
self.assertEquals(['aMessage'], self.observer.calledMethodNames())
def writerMain(writerReactor, statePath, luceneserverPort, gatewayPort, quickCommit=False):
http11Request = be(
(HttpRequest1_1(),
(SocketPool(reactor=writerReactor, unusedTimeout=5, limits=dict(totalSize=100, destinationSize=10)),),
)
)
indexCommitTimeout = 30
defaultLuceneSettings = LuceneSettings(
commitTimeout=indexCommitTimeout,
readonly=False,
)
luceneWriter = luceneAndReaderConfig(defaultLuceneSettings, http11Request, luceneserverPort)
periodicDownload = PeriodicDownload(
writerReactor,
host='localhost',
port=gatewayPort,
schedule=Schedule(period=1 if quickCommit else 10), # WST: Interval in seconds before sending a new request to the GATEWAY in case of an error while processing batch records.(default=1). IntegrationTests need 1 second! Otherwise tests will fail!
name='index',
autoStart=True)
oaiDownload = OaiDownloadProcessor(
path='/oaix',
metadataPrefix=NORMALISED_DOC_NAME,
workingDirectory=join(statePath, 'harvesterstate', 'gateway'),
userAgentAddition='idx-server',
xWait=True,
name='index',
autoCommit=False)
# Post commit naar Lucene(server):
scheduledCommitPeriodicCall = be(
(PeriodicCall(writerReactor, message='commit', name='Scheduled commit', schedule=Schedule(period=1 if quickCommit else 300), initialSchedule=Schedule(period=1)), # WST: Flushes data from memory to disk. IntegrationTests need 1 second! Otherwise tests will fail! (API).
(AllToDo(), # broadcast message to all components, despite of what kind of message...
# (periodicDownload,), # WST: periodicDownload does not do anything with a 'commit' message? So why send it to it???
(LuceneCommit(host='localhost', port=luceneserverPort,), # 'commit' message results in http post to /commit/ to Lucene server:
# (LogComponent("PERIODIC"),#), # [PERIODIC] httprequest1_1(*(), **{'body': None, 'host': 'localhost', 'request': '/commit/', 'port': 52501, 'method': 'POST'})
(http11Request,),
# ),
)
)
)
)
writerServer = \
(Observable(),
(scheduledCommitPeriodicCall,), # Stuur periodiek een 'Commit' naar de LuceneServer...
# (DebugPrompt(reactor=writerReactor, port=readerPort-1, globals=locals()),),
(periodicDownload, # Ga/connect (periodiek) naar de Gateway-server...
(XmlParseLxml(fromKwarg="data", toKwarg="lxmlNode", parseOptions=dict(huge_tree=True, remove_blank_text=True)),
(oaiDownload, # Haal OAI spulletjes van de Gateway...
(UpdateAdapterFromOaiDownloadProcessor(), # Maakt van een SRU update/delete bericht (lxmlNode) een relevante message: 'delete' of 'add' message.
# (LogComponent("SRU harvest van GATEWAY"),), #[SRU harvest van GATEWAY] add(*(), **{'partname': 'record', 'identifier': 'meresco:record:1', 'lxmlNode': '_ElementTree(<record xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><header><identifier>meresco:record:1</identifier><datestamp>2016-07-13T15:31:10Z</datestamp></header><metadata><document xmlns="http://meres
(FilterMessages(allowed=['add']),
(XmlXPath(['/oai:record/oai:metadata/document:document'], fromKwarg='lxmlNode'),
# (LogComponent("NormdocToFieldsList"),),
(NormdocToFieldsList(), # Platte lijst met veldnamen en waardes...
(RecordPidToAuthNid(),),
# (LogComponent("NormdocToFieldsList"),), # [DcToFieldsList] add(*(), **{'fieldslist': [('dc:identifier', 'http://meresco.com?record=1'), ('dc:description', 'This is an example program about Search with Meresco'), ('dc:title', 'Example Program 1'), ('dc:creator', 'Seecr'), ('dc:publisher', 'Seecr'), ('dc:date', '2016'), ('dc:type', 'Example'), ('dc:subject', 'Search'), ('dc:language', 'en'), ('dc:rights', 'Open Source')], 'partname': 'record', 'identifier': 'meresco:record:1'})
# [NormdocToFieldsList] lookupNameIds(*(set(['doi:10.1002/lno.10611', 'wos:000423029300003']),), **{})
(FieldsListToLuceneDocument( # Maakt addDocument messege + creeert de facet/drilldown velden waarvan de value's tot max. 256 chars getruncated worden.
fieldRegistry=luceneWriter.settings.fieldRegistry, # o.a. drilldownfields definitie
untokenizedFieldnames=untokenizedFieldnames, # untokenized fields
indexFieldFactory=DcFields, # Creeert een "__all__", veldnaam en optioneel "untokenized.veldnaam"...
#rewriteIdentifier=(lambda idee: idee.split(':', 1)[-1]) # meresco:record:1' => 'record:1'
),
# (LogComponent("FieldsListToLuceneDocument"),), # [LUCENE_WRITER] addDocument(*(), **{'fields': [{'type': 'TextField', 'name': '__all__', 'value': 'http://meresco.com?record=1'}, {'type': 'TextField', 'name': 'dc:identifier', 'value': 'http://meresco.com?record=1'}, {'type': 'StringField', 'name': 'untokenized.dc:identifier', 'value': 'http://meresco.com?record=1'}, {'type': 'TextField', 'name': '__all__', 'value': 'This is an example program about Search with Meresco'}, {'type': 'TextField', 'name': 'dc:description', 'value': 'This is an example program about Search with Meresco'}, {'type': 'TextField', 'name': '__all__', 'value': 'Example Program 1'}, {'type': 'TextField', 'name': 'dc:title', 'value': 'Example Program 1'}, {'type': 'TextField', 'name': '__all__', 'value': 'Seecr'}, {'type': 'TextField', 'name': 'dc:creator', 'value': 'Seecr'}, {'type': 'TextField', 'name': '__all__', 'value': 'Seecr'}, {'type': 'TextField', 'name': 'dc:publisher', 'value': 'Seecr'}, {'type': 'TextField', 'name': '__all__', 'value': '2016'}, {'type': 'TextField', 'name': 'dc:date', 'value': '2016'}, {'path': ['2016'], 'type': 'FacetField', 'name': 'untokenized.dc:date'}, {'type': 'TextField', 'name': '__all__', 'value': 'Example'}, {'type': 'TextField', 'name': 'dc:type', 'value': 'Example'}, {'type': 'TextField', 'name': '__all__', 'value': 'Search'}, {'type': 'TextField', 'name': 'dc:subject', 'value': 'Search'}, {'path': ['Search'], 'type': 'FacetField', 'name': 'untokenized.dc:subject'}, {'type': 'TextField', 'name': '__all__', 'value': 'en'}, {'type': 'TextField', 'name': 'dc:language', 'value': 'en'}, {'type': 'TextField', 'name': '__all__', 'value': 'Open Source'}, {'type': 'TextField', 'name': 'dc:rights', 'value': 'Open Source'}], 'identifier': 'meresco:record:1'})
# [####LUCENE_WRITER] addDocument(*(), **{'fields': [{'type': 'TextField', 'name': '__all__', 'value': 'knaw'}, {'type': 'TextField', 'name': 'meta:id', 'value': 'knaw'}, {'type': 'TextField', 'name': '__all__', 'value': 'olddata'}, {'type': 'TextField', 'name': 'meta:set', 'value': 'olddata'}, {'type': 'TextField', 'name': '__all__', 'value': 'http://oai.knaw.nl/oai'}, {'type': 'TextField', 'name': 'meta:baseurl', 'value': 'http://oai.knaw.nl/oai'}, {'type': 'TextField', 'name': '__all__', 'value': 'knaw'}, {'type': 'TextField', 'name': 'meta:repositoryGroupId', 'value': 'knaw'}, {'type': 'TextField', 'name': '__all__', 'value': 'nl_didl'}, {'type': 'TextField', 'name': 'meta:metadataPrefix', 'value': 'nl_didl'}, {'type': 'TextField', 'name': '__all__', 'value': 'publication'}, {'type': 'TextField', 'name': 'meta_collection', 'value': 'publication'}, {'path': ['publication'], 'type': 'FacetField', 'name': 'untokenized.meta_collection'}], 'identifier': 'knaw:record:3'})
(luceneWriter,),
# ),
)
)
# )
# )
)
),
(FilterMessages(allowed=['delete']),
(luceneWriter,),
)
)
)
)
)
)
return writerServer
