def testAutoStartOrScheduleRequired(self): reactor = CallTrace('reactor') self.assertRaises(ValueError, lambda: PeriodicCall(reactor=reactor)) self.assertRaises( ValueError, lambda: PeriodicCall(reactor=reactor, autoStart=True)) try: PeriodicCall(reactor=reactor, autoStart=False) except: self.fail('Unexpected exception') try: PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=False) except: self.fail('Unexpected exception') try: PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True) except: self.fail('Unexpected exception')
def newDNA(self, **kwargs): emptyGeneratorMethods = [kwargs.get('message', 'handle')] self.reactor = CallTrace('Reactor', returnValues={'addTimer': 'TOKEN'}) self.observer = CallTrace('Observer', emptyGeneratorMethods=emptyGeneratorMethods, ignoredAttributes=['observer_init']) self.pc = PeriodicCall(self.reactor, **kwargs) self.dna = be((Observable(), (self.pc, (self.observer,), ), ))
def testPausePausesOnStart(self): # autoStart reactor = CallTrace('reactor') pc = PeriodicCall(reactor=reactor, autoStart=False) pc.observer_init() self.assertEquals([], reactor.calledMethodNames()) # explicit .pause() pc = PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True) pc.pause() pc.observer_init() self.assertEquals([], reactor.calledMethodNames())
def jvmMonitorTree(reactor, updatableGustosClient): gustosInterval = 1 if __builtins__.get('__test__', False) else 60 return be(( PeriodicCall(reactor=reactor, schedule=Schedule(period=gustosInterval)), ( JvmMonitor(), (updatableGustosClient, ), ), ))
class PeriodicCallTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.newDNA(schedule=Schedule(period=3600), errorSchedule=Schedule(period=15), prio=9, name='obs_name') list(compose(self.dna.once.observer_init())) def newDNA(self, **kwargs): emptyGeneratorMethods = [kwargs.get('message', 'handle')] self.reactor = CallTrace('Reactor', returnValues={'addTimer': 'TOKEN'}) self.observer = CallTrace('Observer', emptyGeneratorMethods=emptyGeneratorMethods, ignoredAttributes=['observer_init']) self.pc = PeriodicCall(self.reactor, **kwargs) self.dna = be((Observable(), (self.pc, (self.observer,), ), )) def testWithoutData(self): self.assertEquals('obs_name', self.pc.observable_name()) self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs)) # addTimer(3600, pc._periodicCall) self.reactor.calledMethods.reset() addTimer.args[1]() self.assertEquals(['addProcess'], self.reactor.calledMethodNames()) addProcess, = self.reactor.calledMethods self.assertEquals((1, 1), (len(addProcess.args), len(addProcess.kwargs))) self.assertEquals({'prio': 9}, addProcess.kwargs) thisNext = addProcess.args[0] self.reactor.calledMethods.reset() self.assertEquals([], self.observer.calledMethodNames()) thisNext() self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) removeProcess, addTimer = self.reactor.calledMethods self.assertEquals(((), {}), (removeProcess.args, removeProcess.kwargs)) self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs)) def testWithoutDataWithDefaultValues(self): self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs)) self.reactor.calledMethods.reset() addTimer.args[1]() self.assertEquals(['addProcess'], self.reactor.calledMethodNames()) addProcess, = self.reactor.calledMethods self.assertEquals((1, 1), (len(addProcess.args), len(addProcess.kwargs))) self.assertEquals({'prio': 9}, addProcess.kwargs) thisNext = addProcess.args[0] self.reactor.calledMethods.reset() self.assertEquals([], self.observer.calledMethodNames()) thisNext() self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) removeProcess, addTimer = self.reactor.calledMethods self.assertEquals(((), {}), (removeProcess.args, removeProcess.kwargs)) self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs)) def testAutoStartOrScheduleRequired(self): reactor = CallTrace('reactor') self.assertRaises(ValueError, lambda: PeriodicCall(reactor=reactor)) self.assertRaises(ValueError, lambda: PeriodicCall(reactor=reactor, autoStart=True)) try: PeriodicCall(reactor=reactor, autoStart=False) except: self.fail('Unexpected exception') try: PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=False) except: self.fail('Unexpected exception') try: PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True) except: self.fail('Unexpected exception') def testInitialSchedule(self): self.newDNA(initialSchedule=Schedule(period=0), schedule=Schedule(period=12)) list(compose(self.dna.once.observer_init())) self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() self.assertEquals(0, addTimer.args[0]) addTimer.args[1]() addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() addProcess.args[0]() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) removeProcess, addTimer = self.reactor.calledMethods self.assertEquals(12, addTimer.args[0]) def testSetSchedule(self): addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() addTimer.args[1]() self.pc.setSchedule(schedule=Schedule(period=1)) addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() addProcess.args[0]() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) removeProcess, addTimer = self.reactor.calledMethods self.assertEquals(1, addTimer.args[0]) def testSetScheduleWithIdenticalScheduleDoesNothing(self): addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() self.assertEquals(3600, addTimer.args[0]) self.pc.setSchedule(schedule=Schedule(period=3600)) self.assertEquals([], self.reactor.calledMethodNames()) def testSetScheduleAddsANewTimer(self): self.reactor.calledMethods.reset() self.pc.setSchedule(schedule=Schedule(period=123)) self.assertEquals(['removeTimer', 'addTimer'], self.reactor.calledMethodNames()) removeTimer, addTimer = self.reactor.calledMethods self.assertEquals('TOKEN', removeTimer.args[0]) self.assertEquals(123, addTimer.args[0]) def testErrorIntervalAndLoggedMessage(self): def raiser(): raise Exception('exception') yield self.observer.methods['handle'] = raiser addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() addTimer.args[1]() addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() with stderr_replaced() as err: addProcess.args[0]() errValue = err.getvalue() self.assertTrue(errValue.startswith(repr(self.pc))) self.assertTrue('Traceback' in errValue, errValue) self.assertTrue('Exception: exception' in errValue, errValue) self.assertEquals('exception', self.pc.getState().errorState) self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) _, addTimer = self.reactor.calledMethods self.assertEquals(((15, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs)) def testErrorStateReset(self): handleCalledBefore = [] def handle(): if not handleCalledBefore: handleCalledBefore.append(True) raise Exception('exception') yield Yield self.observer.methods['handle'] = handle addTimer, = self.reactor.calledMethods timerCallback = addTimer.args[1] self.reactor.calledMethods.reset() timerCallback() addProcess, = self.reactor.calledMethods addProcessCallback = addProcess.args[0] self.reactor.calledMethods.reset() with stderr_replaced(): addProcessCallback() self.assertEquals('exception', self.pc.getState().errorState) self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) _, addTimer = self.reactor.calledMethods timerCallback = addTimer.args[1] self.reactor.calledMethods.reset() timerCallback() addProcess, = self.reactor.calledMethods addProcessCallback = addProcess.args[0] self.reactor.calledMethods.reset() addProcessCallback() self.assertEquals(['handle', 'handle'], self.observer.calledMethodNames()) addProcessCallback() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) self.assertEquals(None, self.pc.getState().errorState) def testFatalErrorReRaised(self): for exception in [KeyboardInterrupt, SystemExit, AssertionError]: self.newDNA(schedule=Schedule(period=987)) def raiser(): raise exception('msg') yield self.observer.methods['handle'] = raiser list(compose(self.dna.once.observer_init())) addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() addTimer.args[1]() addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() try: addProcess.args[0]() except exception: pass else: self.fail() self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) def testHandleWithCallableAndData(self): suspend = CallTrace('Suspend', returnValues={'__call__': lambda *a, **kw: None}) handleLog = [] def handle(): handleLog.append('ignored-da') yield 'ignored-da' handleLog.append('Suspend') yield suspend handleLog.append('ignored-ta') yield 'ignored-ta' handleLog.append('Stop') self.observer.methods['handle'] = handle addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() addTimer.args[1]() addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() # 'ignored-da' self.assertEquals([], handleLog) addProcess.args[0]() self.assertEquals(['handle'], self.observer.calledMethodNames()) handleCall, = self.observer.calledMethods self.assertEquals(((), {}), (handleCall.args, handleCall.kwargs)) self.assertEquals(['ignored-da'], handleLog) self.assertEquals([], self.reactor.calledMethodNames()) self.observer.calledMethods.reset() self.reactor.calledMethods.reset() # suspend (suspend(reactor, this.next)) self.assertEquals([], suspend.calledMethodNames()) addProcess.args[0]() self.assertEquals(['ignored-da', 'Suspend'], handleLog) self.assertEquals([], self.observer.calledMethodNames()) self.assertEquals([], self.reactor.calledMethodNames()) self.assertEquals(['__call__'], suspend.calledMethodNames()) dunderCall, = suspend.calledMethods self.assertEquals(2, len(dunderCall.args)) self.assertEquals(self.reactor, dunderCall.args[0]) self.assertEquals(GeneratorType, type(dunderCall.args[1].__self__)) self.assertEquals('_periodicCall', dunderCall.args[1].__self__.gi_frame.f_code.co_name) self.assertEquals({}, dunderCall.kwargs) self.observer.calledMethods.reset() self.reactor.calledMethods.reset() suspend.calledMethods.reset() # suspend (suspend.resumeProcess()) dunderCall.args[1]() self.assertEquals(['resumeProcess'], suspend.calledMethodNames()) resumeProcess, = suspend.calledMethods self.assertEquals(((), {}), (resumeProcess.args, resumeProcess.kwargs)) self.assertEquals([], self.reactor.calledMethodNames()) self.assertEquals([], self.observer.calledMethodNames()) self.observer.calledMethods.reset() self.reactor.calledMethods.reset() suspend.calledMethods.reset() # 'ignored-ta' addProcess.args[0]() self.assertEquals(['ignored-da', 'Suspend', 'ignored-ta'], handleLog) self.assertEquals([], suspend.calledMethodNames()) self.assertEquals([], self.reactor.calledMethodNames()) self.assertEquals([], self.observer.calledMethodNames()) # 'Stop' addProcess.args[0]() self.assertEquals(['ignored-da', 'Suspend', 'ignored-ta', 'Stop'], handleLog) self.assertEquals([], suspend.calledMethodNames()) self.assertEquals([], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) removeProcess, addTimer = self.reactor.calledMethods self.assertEquals(((), {}), (removeProcess.args, removeProcess.kwargs)) self.assertEquals(((3600, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs)) @stderr_replaced def testPausePausesOnStart(self): # autoStart reactor = CallTrace('reactor') pc = PeriodicCall(reactor=reactor, autoStart=False) pc.observer_init() self.assertEquals([], reactor.calledMethodNames()) # explicit .pause() pc = PeriodicCall(reactor=reactor, schedule=Schedule(period=1), autoStart=True) pc.pause() pc.observer_init() self.assertEquals([], reactor.calledMethodNames()) @stderr_replaced def testPausePausesWhenRunning(self): self.newDNA(schedule=Schedule(period=1), autoStart=True) list(compose(self.dna.once.observer_init())) self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods # pauses after completing current task self.pc.pause() self.reactor.calledMethods.reset() addTimer.args[1]() self.assertEquals(['addProcess'], self.reactor.calledMethodNames()) addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() addProcess.args[0]() self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess'], self.reactor.calledMethodNames()) def testPauseRemovesInitialPendingTimer(self): self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) self.reactor.calledMethods.reset() with stderr_replaced() as err: self.pc.pause() self.assertEquals('%s: paused\n' % repr(self.pc), err.getvalue()) self.assertEquals(['removeTimer'], self.reactor.calledMethodNames()) removeTimer, = self.reactor.calledMethods self.assertEquals((('TOKEN',), {}), (removeTimer.args, removeTimer.kwargs)) def testPauseRemovesEndOfProcessPendingTimer(self): self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() # Process until timer added again addTimer.args[1]() addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() addProcess.args[0]() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) self.reactor.calledMethods.reset() # pause at pending timer with stderr_replaced() as err: self.pc.pause() self.assertEquals('%s: paused\n' % repr(self.pc), err.getvalue()) self.assertEquals(['removeTimer'], self.reactor.calledMethodNames()) removeTimer, = self.reactor.calledMethods self.assertEquals((('TOKEN',), {}), (removeTimer.args, removeTimer.kwargs)) def testResumeStartsWhenPaused(self): self.newDNA(schedule=Schedule(period=2), autoStart=False) list(compose(self.dna.once.observer_init())) self.assertEquals([], self.reactor.calledMethodNames()) with stderr_replaced() as err: self.pc.resume() self.assertEquals('%s: resumed\n' % repr(self.pc), err.getvalue()) self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.assertEquals(self.pc._periodicCall, addTimer.args[1]) # finish one task (1/2) self.reactor.calledMethods.reset() addTimer.args[1]() addProcess, = self.reactor.calledMethods # finish one task (2/2) self.reactor.calledMethods.reset() addProcess.args[0]() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) @stderr_replaced def testDoNotResumeNotPaused(self): self.newDNA(schedule=Schedule(period=1), autoStart=True) self.pc.resume() self.assertEquals([], self.reactor.calledMethodNames()) list(compose(self.dna.once.observer_init())) self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() self.pc.resume() self.assertEquals([], self.reactor.calledMethodNames()) addTimer.args[1]() self.assertEquals(['addProcess'], self.reactor.calledMethodNames()) self.reactor.calledMethods.reset() self.pc.resume() self.assertEquals([], self.reactor.calledMethodNames()) @stderr_replaced def testResumeOnlyAddsATimerWhenNotBusy(self): # Because at the end of the busy / reactor processing it # will add a timer when not paused. addTimer, = self.reactor.calledMethods # "Firing" timer callback self.reactor.calledMethods.reset() addTimer.args[1]() self.assertEquals(['addProcess'], self.reactor.calledMethodNames()) addProcess, = self.reactor.calledMethods busybusy = [] def handleBusyMock(): yield busybusy.append(True) yield busybusy.append(True) self.observer.methods['handle'] = handleBusyMock # Busy processing, so pause won't work immediately self.reactor.calledMethods.reset() self.pc.pause() # <-- pause self.assertEquals([], self.reactor.calledMethodNames()) self.assertEquals([], busybusy) addProcess.args[0]() # <-- doing stuff self.assertEquals([], self.reactor.calledMethodNames()) self.assertEquals([True], busybusy) self.pc.resume() # <-- resume called addProcess.args[0]() # <-- doing more stuff self.assertEquals([True, True], busybusy) self.assertEquals([], self.reactor.calledMethodNames()) addProcess.args[0]() # <-- done doing and delayed resume (addTimer) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) @stderr_replaced def testGetState(self): state = self.pc.getState() self.assertEquals('obs_name', state.name) self.assertEquals(False, state.paused) self.assertEquals(Schedule(period=3600), state.schedule) self.assertEquals({'paused': False, 'name': 'obs_name', 'schedule': Schedule(period=3600)}, state.asDict()) self.pc.observable_setName('dashy') self.assertEquals('dashy', state.name) self.assertEquals(None, state.errorState) self.pc.setSchedule(schedule=Schedule(period=5)) self.assertEquals(Schedule(period=5), state.schedule) self.pc.pause() self.assertEquals(True, state.paused) self.pc.resume() addTimer = self.reactor.calledMethods[-1] self.reactor.calledMethods.reset() self.assertEquals(False, state.paused) addTimer.args[1]() self.pc.pause() self.assertEquals(False, state.paused) addProcess = self.reactor.calledMethods[-1] addProcess.args[0]() self.assertEquals(True, state.paused) def testRepr(self): self.assertEquals("PeriodicCall(name='obs_name', paused=False, schedule=Schedule(period=3600))", repr(self.pc)) def testShorten(self): message = 'x' * (MAX_LENGTH) self.assertEquals(message, message) message = 'x' * (MAX_LENGTH) + 'z' self.assertNotEqual(message, shorten(message)) self.assertEquals(MAX_LENGTH + len(' ... '), len(shorten(message))) self.assertTrue(' ... ' in shorten(message), shorten(message)) self.assertTrue(shorten(message).startswith('xxxx'), shorten(message)) self.assertTrue(shorten(message).endswith('xxxz'), shorten(message)) def testMessageConfigurable(self): self.newDNA(message="aMessage", schedule=Schedule(period=3600), errorSchedule=Schedule(period=15), prio=9, name='obs_name') list(compose(self.dna.once.observer_init())) self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods callback = addTimer.args[1] self.reactor.calledMethods.reset() callback() self.assertEquals(['addProcess'], self.reactor.calledMethodNames()) addProcess, = self.reactor.calledMethods callback = addProcess.args[0] self.reactor.calledMethods.reset() callback() self.assertEquals(['aMessage'], self.observer.calledMethodNames())
def writerMain(writerReactor, statePath, luceneserverPort, gatewayPort, quickCommit=False): http11Request = be( (HttpRequest1_1(), (SocketPool(reactor=writerReactor, unusedTimeout=5, limits=dict(totalSize=100, destinationSize=10)),), ) ) indexCommitTimeout = 30 defaultLuceneSettings = LuceneSettings( commitTimeout=indexCommitTimeout, readonly=False, ) luceneWriter = luceneAndReaderConfig(defaultLuceneSettings, http11Request, luceneserverPort) periodicDownload = PeriodicDownload( writerReactor, host='localhost', port=gatewayPort, schedule=Schedule(period=1 if quickCommit else 10), # WST: Interval in seconds before sending a new request to the GATEWAY in case of an error while processing batch records.(default=1). IntegrationTests need 1 second! Otherwise tests will fail! name='index', autoStart=True) oaiDownload = OaiDownloadProcessor( path='/oaix', metadataPrefix=NORMALISED_DOC_NAME, workingDirectory=join(statePath, 'harvesterstate', 'gateway'), userAgentAddition='idx-server', xWait=True, name='index', autoCommit=False) # Post commit naar Lucene(server): scheduledCommitPeriodicCall = be( (PeriodicCall(writerReactor, message='commit', name='Scheduled commit', schedule=Schedule(period=1 if quickCommit else 300), initialSchedule=Schedule(period=1)), # WST: Flushes data from memory to disk. IntegrationTests need 1 second! Otherwise tests will fail! (API). (AllToDo(), # broadcast message to all components, despite of what kind of message... # (periodicDownload,), # WST: periodicDownload does not do anything with a 'commit' message? So why send it to it??? (LuceneCommit(host='localhost', port=luceneserverPort,), # 'commit' message results in http post to /commit/ to Lucene server: # (LogComponent("PERIODIC"),#), # [PERIODIC] httprequest1_1(*(), **{'body': None, 'host': 'localhost', 'request': '/commit/', 'port': 52501, 'method': 'POST'}) (http11Request,), # ), ) ) ) ) writerServer = \ (Observable(), (scheduledCommitPeriodicCall,), # Stuur periodiek een 'Commit' naar de LuceneServer... # (DebugPrompt(reactor=writerReactor, port=readerPort-1, globals=locals()),), (periodicDownload, # Ga/connect (periodiek) naar de Gateway-server... (XmlParseLxml(fromKwarg="data", toKwarg="lxmlNode", parseOptions=dict(huge_tree=True, remove_blank_text=True)), (oaiDownload, # Haal OAI spulletjes van de Gateway... (UpdateAdapterFromOaiDownloadProcessor(), # Maakt van een SRU update/delete bericht (lxmlNode) een relevante message: 'delete' of 'add' message. # (LogComponent("SRU harvest van GATEWAY"),), #[SRU harvest van GATEWAY] add(*(), **{'partname': 'record', 'identifier': 'meresco:record:1', 'lxmlNode': '_ElementTree(<record xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><header><identifier>meresco:record:1</identifier><datestamp>2016-07-13T15:31:10Z</datestamp></header><metadata><document xmlns="http://meres (FilterMessages(allowed=['add']), (XmlXPath(['/oai:record/oai:metadata/document:document'], fromKwarg='lxmlNode'), # (LogComponent("NormdocToFieldsList"),), (NormdocToFieldsList(), # Platte lijst met veldnamen en waardes... (RecordPidToAuthNid(),), # (LogComponent("NormdocToFieldsList"),), # [DcToFieldsList] add(*(), **{'fieldslist': [('dc:identifier', 'http://meresco.com?record=1'), ('dc:description', 'This is an example program about Search with Meresco'), ('dc:title', 'Example Program 1'), ('dc:creator', 'Seecr'), ('dc:publisher', 'Seecr'), ('dc:date', '2016'), ('dc:type', 'Example'), ('dc:subject', 'Search'), ('dc:language', 'en'), ('dc:rights', 'Open Source')], 'partname': 'record', 'identifier': 'meresco:record:1'}) # [NormdocToFieldsList] lookupNameIds(*(set(['doi:10.1002/lno.10611', 'wos:000423029300003']),), **{}) (FieldsListToLuceneDocument( # Maakt addDocument messege + creeert de facet/drilldown velden waarvan de value's tot max. 256 chars getruncated worden. fieldRegistry=luceneWriter.settings.fieldRegistry, # o.a. drilldownfields definitie untokenizedFieldnames=untokenizedFieldnames, # untokenized fields indexFieldFactory=DcFields, # Creeert een "__all__", veldnaam en optioneel "untokenized.veldnaam"... #rewriteIdentifier=(lambda idee: idee.split(':', 1)[-1]) # meresco:record:1' => 'record:1' ), # (LogComponent("FieldsListToLuceneDocument"),), # [LUCENE_WRITER] addDocument(*(), **{'fields': [{'type': 'TextField', 'name': '__all__', 'value': 'http://meresco.com?record=1'}, {'type': 'TextField', 'name': 'dc:identifier', 'value': 'http://meresco.com?record=1'}, {'type': 'StringField', 'name': 'untokenized.dc:identifier', 'value': 'http://meresco.com?record=1'}, {'type': 'TextField', 'name': '__all__', 'value': 'This is an example program about Search with Meresco'}, {'type': 'TextField', 'name': 'dc:description', 'value': 'This is an example program about Search with Meresco'}, {'type': 'TextField', 'name': '__all__', 'value': 'Example Program 1'}, {'type': 'TextField', 'name': 'dc:title', 'value': 'Example Program 1'}, {'type': 'TextField', 'name': '__all__', 'value': 'Seecr'}, {'type': 'TextField', 'name': 'dc:creator', 'value': 'Seecr'}, {'type': 'TextField', 'name': '__all__', 'value': 'Seecr'}, {'type': 'TextField', 'name': 'dc:publisher', 'value': 'Seecr'}, {'type': 'TextField', 'name': '__all__', 'value': '2016'}, {'type': 'TextField', 'name': 'dc:date', 'value': '2016'}, {'path': ['2016'], 'type': 'FacetField', 'name': 'untokenized.dc:date'}, {'type': 'TextField', 'name': '__all__', 'value': 'Example'}, {'type': 'TextField', 'name': 'dc:type', 'value': 'Example'}, {'type': 'TextField', 'name': '__all__', 'value': 'Search'}, {'type': 'TextField', 'name': 'dc:subject', 'value': 'Search'}, {'path': ['Search'], 'type': 'FacetField', 'name': 'untokenized.dc:subject'}, {'type': 'TextField', 'name': '__all__', 'value': 'en'}, {'type': 'TextField', 'name': 'dc:language', 'value': 'en'}, {'type': 'TextField', 'name': '__all__', 'value': 'Open Source'}, {'type': 'TextField', 'name': 'dc:rights', 'value': 'Open Source'}], 'identifier': 'meresco:record:1'}) # [####LUCENE_WRITER] addDocument(*(), **{'fields': [{'type': 'TextField', 'name': '__all__', 'value': 'knaw'}, {'type': 'TextField', 'name': 'meta:id', 'value': 'knaw'}, {'type': 'TextField', 'name': '__all__', 'value': 'olddata'}, {'type': 'TextField', 'name': 'meta:set', 'value': 'olddata'}, {'type': 'TextField', 'name': '__all__', 'value': 'http://oai.knaw.nl/oai'}, {'type': 'TextField', 'name': 'meta:baseurl', 'value': 'http://oai.knaw.nl/oai'}, {'type': 'TextField', 'name': '__all__', 'value': 'knaw'}, {'type': 'TextField', 'name': 'meta:repositoryGroupId', 'value': 'knaw'}, {'type': 'TextField', 'name': '__all__', 'value': 'nl_didl'}, {'type': 'TextField', 'name': 'meta:metadataPrefix', 'value': 'nl_didl'}, {'type': 'TextField', 'name': '__all__', 'value': 'publication'}, {'type': 'TextField', 'name': 'meta_collection', 'value': 'publication'}, {'path': ['publication'], 'type': 'FacetField', 'name': 'untokenized.meta_collection'}], 'identifier': 'knaw:record:3'}) (luceneWriter,), # ), ) ) # ) # ) ) ), (FilterMessages(allowed=['delete']), (luceneWriter,), ) ) ) ) ) ) return writerServer