def testApacheLog(self): requestHandler = CallTrace( 'handler', ignoredAttributes=['writeLog', 'do_unknown']) requestHandler.returnValues['handleRequest'] = ( f for f in [Yield, okPlainText, 'te', callable, 'xt']) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), ( LogCollector(), (handleRequestLog, (requestHandler, )), (ApacheLogWriter(stream), ), ))) result = asList( observable.all.handleRequest( Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={ 'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test' }, otherKwarg='value')) self.assertEquals([Yield, okPlainText, 'te', callable, 'xt'], result) self.assertEquals(['handleRequest'], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n', logline)
def testAnalyseLog(self): observer = CallTrace() def handleRequest(**kwargs): yield "We did it!" handleRequestMock = CallTrace(methods={'handleRequest': handleRequest}) dna = be( (Observable(), (LogCollector(), (GustosLogWriter(), (AgentCountReport(gustosGroup="Gustos Group", scopeNames=('mock', )), ), (observer, ) ), (LogCollectorScope("mock"), (HandleRequestLog(), (handleRequestMock, ) ) ) ) ) ) result = list(compose(dna.all.handleRequest(Headers={'User-Agent': GOOGLE_BOT}))) self.assertEqual(b"We did it!", result[0]) valuesKwarg = observer.calledMethods[-1].kwargs['values'] self.assertEqual({'Gustos Group': {'User agents': {'Googlebot/2.1': {'count': 1}}}}, valuesKwarg)
def testCountOne(self): self.report = CallTrace(returnValues={'report': None}, onlySpecifiedMethods=True) self.top = be((Observable(), (LogCollector(), ( GustosLogWriter(), (CountReport(gustosGroup='gustosGroup', scopeNames=(), curveName='calls'), ), (CountReport(gustosGroup='gustosGroup', scopeNames=(), curveName='collectTimes', keys=('collectResponse', 'some')), ), (self.report, ), ), (CollectMeALog(), )))) self.top.do.callMe(times=3) self.assertEqual(['report'], self.report.calledMethodNames()) countReport = self.report.calledMethods[0].kwargs['values'][ 'gustosGroup']['Counts'] self.assertEqual({ 'collectTimes': { 'count': 3 }, 'calls': { 'count': 1 } }, countReport)
def _buildDna(self, defaultRecordSchema="rdf", **kwargs): jsonSearch = JsonSearch(defaultRecordSchema=defaultRecordSchema, **kwargs) jsonSearch._timeNow = self._timeNow self.dna = be( (Observable(), (LogCollector(), (jsonSearch, (self.observer, )))))
def testCountOne(self): self.report = CallTrace(returnValues={'report': { "packet": "data" }}, onlySpecifiedMethods=True) self.top = be((Observable(), (LogCollector(), ( GustosLogWriter(), (ResponseTimeReport(gustosGroup='gustosGroup', scopeNames=(), curveName='http'), ), (ResponseTimeReport(gustosGroup='gustosGroup', scopeNames=(), curveName='other', keys=('otherResponse', 'duration')), ), (self.report, ), ), (CollectMeALog(), )))) self.top.do.callMe(times=3) self.assertEqual(['report'], self.report.calledMethodNames()) theReport = self.report.calledMethods[0].kwargs['values'][ 'gustosGroup']['ResponseTime'] self.assertEqual({ 'other': { 'time': 0.3 }, 'http': { 'time': 0.0 } }, theReport)
def testLogRequestInCaseOfExceptionAfterStatusCode(self): requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown']) def handleRequestRaisesException(**kwargs): yield Yield yield okPlainText yield 'text' raise ValueError('doesntreallymatter') yield requestHandler.methods['handleRequest'] = handleRequestRaisesException stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), (LogCollector(), (handleRequestLog, (requestHandler,) ), (ApacheLogWriter(stream),), ) )) result = [] g = compose(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test'}, otherKwarg='value')) try: for x in g: result.append(x) except Exception: pass self.assertEqual([Yield, bytes(okPlainText, encoding='utf-8'), b'text'], result) self.assertEqual(['handleRequest'], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\')\n', logline)
def testQuery(self): def executeQuery(**kwargs): return Response(total=42) yield index = CallTrace('index', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData'], methods=dict(executeQuery=executeQuery)) observable = be((Observable(), (LogCollector(), (self.handleRequestLog, (SruParser(), (SruHandler(enableCollectLog=True), (index,) ) ) ), (self.queryLogWriter,), ) )) result = asString(observable.all.handleRequest( Method='GET', Client=('127.0.0.1', 1234), arguments={ 'version': ['1.2'], 'operation': ['searchRetrieve'], 'query': ['query'], 'maximumRecords': ['0'], }, path='/path/sru', otherKwarg='value')) self.assertTrue('<srw:numberOfRecords>42</srw:numberOfRecords>' in result, result) self.assertTrue(isfile(join(self.tempdir, '2009-11-02-query.log'))) with open(join(self.tempdir, '2009-11-02-query.log')) as fp: self.assertEqual('2009-11-02T11:25:37Z 127.0.0.1 0.7K 1.000s 42hits /path/sru maximumRecords=0&operation=searchRetrieve&query=query&recordPacking=xml&recordSchema=dc&startRecord=1&version=1.2\n', fp.read())
def dna(reactor, port, dynamic, static, verbose=True): return (Observable(), (ObservableHttpServer(reactor, port=port), (LogCollector(), (ApacheLogWriter(stdout if verbose else None), ), (HandleRequestLog(), (BasicHttpHandler(), (PathFilter('/static'), (PathRename(lambda path: path[len('/static'):]), (FileServer(static), ))), (PathFilter('/', excluding=['/static']), (DynamicHtml([dynamic], reactor=reactor, indexPage='/index'), )))))))
def testWriteLogMustNotFail(self): logwriter = CallTrace('logwriter', emptyGeneratorMethods=['someMessage']) logwriter.exceptions['writeLog'] = ValueError top = be((Observable(), (LogCollector('default'), (logwriter, ), (FilterMessages(allowed=['someMessage']), (LogKeyValue(dict(name='A')), ))))) with stderr_replaced() as err: try: consume(top.all.someMessage()) except ValueError: self.fail("Should not raise an error; Only print it") self.assertTrue('ValueError' in err.getvalue(), err.getvalue())
def testTransparentWontLogIfNothingPresentAll(self): observer = createObserver() observable = be((Observable(), (LogCollector(), (observer, )))) result = asString(observable.all.allMessage('arg', kwarg='kwarg')) self.assertEqual('allresult', result) result = retval(observable.any.anyMessage('arg', kwarg='kwarg')) self.assertEqual('anyresult', result) observable.do.doMessage('arg', kwarg='kwarg') result = observable.call.callMessage('arg', kwarg='kwarg') self.assertEqual('callresult', result) self.assertEqual( ['allMessage', 'anyMessage', 'doMessage', 'callMessage'], observer.calledMethodNames()) for m in observer.calledMethods: self.assertEqual(('arg', ), m.args) self.assertEqual(dict(kwarg='kwarg'), m.kwargs)
def testEmptyQuery(self): requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown']) requestHandler.returnValues['handleRequest'] = (f for f in [okXml, '<sru>', '</sru>']) observable = be((Observable(), (LogCollector(), (self.handleRequestLog, (requestHandler,) ), (self.queryLogWriter,), ) )) result = asString(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), arguments={}, path='/path/sru', otherKwarg='value')) self.assertEquals(okXml+'<sru></sru>', result) self.assertTrue(isfile(join(self.tempdir, '2009-11-02-query.log'))) self.assertEquals('2009-11-02T11:25:37Z 127.0.0.1 0.1K 1.000s - /path/sru \n', open(join(self.tempdir, '2009-11-02-query.log')).read())
def _buildDna(self, defaultRecordSchema="rdf", **kwargs): jsonSearch = JsonSearch(defaultRecordSchema=defaultRecordSchema, enableCollectLog=True, **kwargs) jsonSearch._timeNow = self._timeNow class WriteLog: def writeLog(mii, collectedLog): self.logs.append(collectedLog) self.dna = be( (Observable(), (LogCollector(), (jsonSearch, (self.observer, ) ), (WriteLog(),), ) ) )
def testScope(self): logwriter = CallTrace('logwriter', emptyGeneratorMethods=['someMessage']) top = be( (Observable(), (LogCollector('default'), (logwriter, ), (FilterMessages(allowed=['someMessage']), ( LogKeyValue(dict(name='A')), (LogKeyValue(dict(name='B')), ), ( LogCollectorScope('scope_one'), (LogKeyValue(dict(name='C')), ), ), (LogKeyValue(dict(name='D')), ), ( LogCollectorScope('scope_two'), ( LogKeyValue(dict(name='E')), ( LogCollectorScope('scope_two_one'), (LogKeyValue(dict(name='F')), ), ), ), (LogKeyValue(dict(name='G')), ), ), ))))) consume(top.all.someMessage()) self.assertEqual(['someMessage', 'writeLog'], logwriter.calledMethodNames()) self.assertEqual( { 'name': ['A', 'B', 'D'], 'scope_one': { 'name': ['C'], }, 'scope_two': { 'name': ['E', 'G'], 'scope_two_one': { 'name': ['F'] } } }, logwriter.calledMethods[-1].kwargs['collectedLog'])
def testAlmostNoData(self): # No data due to HandleRequestLog is not used. requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown']) def handleRequest(**kwarg): collectLog(dict(key='value')) yield okXml yield '<sru></sru>' requestHandler.methods['handleRequest'] = handleRequest observable = be((Observable(), (LogCollector(), (requestHandler,), (self.queryLogWriter,), ) )) result = asString(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), arguments={}, path='/path/sru', otherKwarg='value')) self.assertEqual(okXml+'<sru></sru>', result) self.assertEqual(0, len(listdir(self.tempdir)))
def testOne(self): def fillReport(groups, collectedLog): if not 'count' in groups: groups['count'] = 0 groups['count'] += collectedLog['count'][0] reports = [] def report(values): reports.append(values.copy()) observer = CallTrace(methods={ 'fillReport': fillReport, 'report': report }) reactor = StoppableReactor() class SomethingCounter(object): def something(this): collectLog(dict(count=1)) def reactorShutdown(): reactor.stop() reactor.addTimer(2.5, reactorShutdown) dna = be((Observable(), (LogCollector(), (GustosContinuousLogWriter(reactor=reactor, interval=1.0), (observer, )), (SomethingCounter(), )))) def job(): dna.do.something() reactor.addTimer(0.5, job) job() reactor.loop() self.assertEqual([ 'analyseLog', 'fillReport', 'analyseLog', 'fillReport', 'report', 'analyseLog', 'fillReport', 'analyseLog', 'fillReport', 'report', 'analyseLog', 'fillReport' ], [m.name for m in observer.calledMethods]) self.assertEqual([{'count': 2}, {'count': 4}], reports)
def testLog(self): class SomeLog(Transparent): def logMe(self, argument): collectLog(dict(logArgument=argument)) return self.call.logMe(argument=argument) observer = createObserver() observer.methods['doNotLogMe'] = observer.methods['logMe'] = observer.methods['callMessage'] observable = be((Observable(), (LogCollector(), (SomeLog(), (observer,) ), ) )) self.assertEquals('callresult', observable.call.logMe(argument=0)) self.assertEquals('callresult', observable.call.doNotLogMe(argument=0)) self.assertEquals(['logMe', 'writeLog', 'doNotLogMe'], observer.calledMethodNames()) writeLog = observer.calledMethods[1] self.assertEquals((), writeLog.args) self.assertEquals(['collectedLog'], writeLog.kwargs.keys()) self.assertEquals({'logArgument': [0]}, writeLog.kwargs['collectedLog'])
def testLogHttpError(self): requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown']) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), (LogCollector(), (handleRequestLog, (requestHandler,), ), (ApacheLogWriter(stream),), ) )) # called by ObservableHttpServer observable.do.logHttpError(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value') logline = stream.getvalue() self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 503 - "-" "-"\n', logline) self.assertEqual(['logHttpError'], requestHandler.calledMethodNames()) self.assertEqual(dict(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value'), requestHandler.calledMethods[0].kwargs)
def testLoggedPathsNewStyle(self): log = CallTrace('log') def handleRequest(**kwargs): yield okPlainText yield 'result' index = CallTrace('index', methods={'handleRequest': handleRequest}) observable = be((Observable(), (LogCollector(), (QueryLogWriter(log=log, scopeNames=('global', 'yesPath')), ), (LogCollectorScope('global'), (HandleRequestLog(), (PathFilter('/yes'), ( LogCollectorScope('yesPath'), (index, ), )), ( PathFilter('/no'), (index, ), )))))) result = asString( observable.all.handleRequest(Client=('11.22.33.44', 1234), path='/yes')) self.assertEquals(okPlainText + 'result', result) result = asString( observable.all.handleRequest(Client=('22.33.44.55', 2345), path='/no')) self.assertEquals(okPlainText + 'result', result) result = asString( observable.all.handleRequest(Client=('33.44.55.66', 3456), path='/yes')) self.assertEquals(okPlainText + 'result', result) self.assertEquals(['log', 'log'], log.calledMethodNames()) self.assertEquals(['/yes', '/yes'], [m.kwargs['path'] for m in log.calledMethods])
def dna(reactor, port, dataPath, logPath, statePath, externalUrl, customerLogoUrl, deproxyIps=None, **ignored): environment = createEnvironment(dataPath) harvesterData = environment.createHarvesterData() harvesterDataRetrieve = environment.createHarvesterDataRetrieve() deproxy = Deproxy(deproxyForIps=deproxyIps) repositoryStatus = be( (RepositoryStatus(logPath, statePath), (harvesterData, ) ) ) configDict = JsonDict( logPath=logPath, statePath=statePath, externaUrl=externalUrl, dataPath=dataPath, ) print("Started Metastreams with configuration:\n" + configDict.pretty_print()) userGroup = initializeUserGroupManagement(join(statePath, 'users'), harvesterData) basicHtmlLoginHelix = (BasicHtmlLoginForm( action="/login.action", loginPath="/login", home="/index", rememberMeCookie=False, lang="nl"), (userGroup.basicHtmlObserver,), ) varWwwdataPath = join(statePath, 'www-data', 'var') isdir(varWwwdataPath) or makedirs(varWwwdataPath) staticFilePaths = [] staticFiles = Transparent() for path, libdir in [ ('/js/bootstrap', '/usr/share/javascript/bootstrap5/js'), ('/css/bootstrap', '/usr/share/javascript/bootstrap5/css'), ('/css/bootstrap-icons', '/usr/share/javascript/bootstrap-icons'), ('/js/jquery', '/usr/share/javascript/jquery'), ('/js/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter'), ('/css/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter/css'), ('/js/autosize', '/usr/share/javascript/autosize'), ('/static', staticHtmlPath), ('/var', varWwwdataPath), ]: staticFiles.addObserver(StaticFiles(libdir=libdir, path=path)) staticFilePaths.append(path) return \ (Observable(), (ObservableHttpServer(reactor, port), (LogCollector(), (ApacheLogWriter(stdout),), (deproxy, (HandleRequestLog(), (BasicHttpHandler(), (SessionHandler(), (CookieMemoryStore(name="meresco-harvester", timeout=2*60*60), ), (UserFromSession(), (PathFilter("/info/version"), (StringServer(VERSION_STRING, ContentTypePlainText), ) ), (PathFilter("/info/config"), (StringServer(configDict.dumps(), ContentTypeJson), ) ), (PathFilter('/login.action'), basicHtmlLoginHelix ), (staticFiles,), (PathFilter('/', excluding=['/info/version', '/info/config', '/action', '/login.action'] + harvesterDataRetrieve.paths + staticFilePaths), (SecureZone("/login", excluding=["/index", "/invalid", "/rss", '/running.rss', '/showHarvesterStatus'], defaultLanguage="nl"), (PathFilter('/', excluding=userGroup.excludedPaths), (DynamicHtml( [dynamicHtmlPath], reactor=reactor, additionalGlobals={ 'externalUrl': externalUrl, 'escapeXml': escapeXml, 'compose': compose, 'dumps': dumps, 'VERSION': VERSION, 'CONFIG': configDict, 'Timeslot': Timeslot, 'ThroughputAnalyser': ThroughputAnalyser, 'dateSince': dateSince, 'callable': callable, 'OnlineHarvest': OnlineHarvest, 'StringIO': StringIO, 'okPlainText': okPlainText, 'ZuluTime': ZuluTime, 'xpathFirst': xpathFirst, 'customerLogoUrl': customerLogoUrl, 'uuid': lambda: str(uuid4()), }, indexPage="/index", ), basicHtmlLoginHelix, (harvesterData,), (repositoryStatus,), (userGroup.dynamicHtmlObserver,), ) ), (userGroup.actions,), ), ), (PathFilter('/action'), (HarvesterDataActions(), (harvesterData,) ), ), (PathFilter(harvesterDataRetrieve.paths), (harvesterDataRetrieve, (FilterFields(), (harvesterData,), ), (repositoryStatus,), ) ) ) ) ) ) ) ) ) )