def testApacheLog(self):
requestHandler = CallTrace(
'handler', ignoredAttributes=['writeLog', 'do_unknown'])
requestHandler.returnValues['handleRequest'] = (
f for f in [Yield, okPlainText, 'te', callable, 'xt'])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(), (
LogCollector(),
(handleRequestLog, (requestHandler, )),
(ApacheLogWriter(stream), ),
)))
result = asList(
observable.all.handleRequest(
Method='GET',
Client=('127.0.0.1', 1234),
RequestURI='http://example.org/path?key=value',
query='key=value',
path='/path',
Headers={
'Referer': 'http://meresco.org',
'User-Agent': 'Meresco-Components Test'
},
otherKwarg='value'))
self.assertEquals([Yield, okPlainText, 'te', callable, 'xt'], result)
self.assertEquals(['handleRequest'],
requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n',
logline)
def testAnalyseLog(self):
observer = CallTrace()
def handleRequest(**kwargs):
yield "We did it!"
handleRequestMock = CallTrace(methods={'handleRequest': handleRequest})
dna = be(
(Observable(),
(LogCollector(),
(GustosLogWriter(),
(AgentCountReport(gustosGroup="Gustos Group", scopeNames=('mock', )), ),
(observer, )
),
(LogCollectorScope("mock"),
(HandleRequestLog(),
(handleRequestMock, )
)
)
)
)
)
result = list(compose(dna.all.handleRequest(Headers={'User-Agent': GOOGLE_BOT})))
self.assertEqual(b"We did it!", result[0])
valuesKwarg = observer.calledMethods[-1].kwargs['values']
self.assertEqual({'Gustos Group': {'User agents': {'Googlebot/2.1': {'count': 1}}}}, valuesKwarg)
def testCountOne(self):
self.report = CallTrace(returnValues={'report': None},
onlySpecifiedMethods=True)
self.top = be((Observable(), (LogCollector(), (
GustosLogWriter(),
(CountReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='calls'), ),
(CountReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='collectTimes',
keys=('collectResponse', 'some')), ),
(self.report, ),
), (CollectMeALog(), ))))
self.top.do.callMe(times=3)
self.assertEqual(['report'], self.report.calledMethodNames())
countReport = self.report.calledMethods[0].kwargs['values'][
'gustosGroup']['Counts']
self.assertEqual({
'collectTimes': {
'count': 3
},
'calls': {
'count': 1
}
}, countReport)
def _buildDna(self, defaultRecordSchema="rdf", **kwargs):
jsonSearch = JsonSearch(defaultRecordSchema=defaultRecordSchema,
**kwargs)
jsonSearch._timeNow = self._timeNow
self.dna = be(
(Observable(), (LogCollector(), (jsonSearch, (self.observer, )))))
def testCountOne(self):
self.report = CallTrace(returnValues={'report': {
"packet": "data"
}},
onlySpecifiedMethods=True)
self.top = be((Observable(), (LogCollector(), (
GustosLogWriter(),
(ResponseTimeReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='http'), ),
(ResponseTimeReport(gustosGroup='gustosGroup',
scopeNames=(),
curveName='other',
keys=('otherResponse', 'duration')), ),
(self.report, ),
), (CollectMeALog(), ))))
self.top.do.callMe(times=3)
self.assertEqual(['report'], self.report.calledMethodNames())
theReport = self.report.calledMethods[0].kwargs['values'][
'gustosGroup']['ResponseTime']
self.assertEqual({
'other': {
'time': 0.3
},
'http': {
'time': 0.0
}
}, theReport)
def testLogRequestInCaseOfExceptionAfterStatusCode(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
def handleRequestRaisesException(**kwargs):
yield Yield
yield okPlainText
yield 'text'
raise ValueError('doesntreallymatter')
yield
requestHandler.methods['handleRequest'] = handleRequestRaisesException
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(),
(LogCollector(),
(handleRequestLog,
(requestHandler,)
),
(ApacheLogWriter(stream),),
)
))
result = []
g = compose(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test'}, otherKwarg='value'))
try:
for x in g:
result.append(x)
except Exception:
pass
self.assertEqual([Yield, bytes(okPlainText, encoding='utf-8'), b'text'], result)
self.assertEqual(['handleRequest'], requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\')\n', logline)
def testQuery(self):
def executeQuery(**kwargs):
return Response(total=42)
yield
index = CallTrace('index',
emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData'],
methods=dict(executeQuery=executeQuery))
observable = be((Observable(),
(LogCollector(),
(self.handleRequestLog,
(SruParser(),
(SruHandler(enableCollectLog=True),
(index,)
)
)
),
(self.queryLogWriter,),
)
))
result = asString(observable.all.handleRequest(
Method='GET',
Client=('127.0.0.1', 1234),
arguments={
'version': ['1.2'],
'operation': ['searchRetrieve'],
'query': ['query'],
'maximumRecords': ['0'],
},
path='/path/sru',
otherKwarg='value'))
self.assertTrue('<srw:numberOfRecords>42</srw:numberOfRecords>' in result, result)
self.assertTrue(isfile(join(self.tempdir, '2009-11-02-query.log')))
with open(join(self.tempdir, '2009-11-02-query.log')) as fp:
self.assertEqual('2009-11-02T11:25:37Z 127.0.0.1 0.7K 1.000s 42hits /path/sru maximumRecords=0&operation=searchRetrieve&query=query&recordPacking=xml&recordSchema=dc&startRecord=1&version=1.2\n', fp.read())
def dna(reactor, port, dynamic, static, verbose=True):
return (Observable(), (ObservableHttpServer(reactor, port=port),
(LogCollector(),
(ApacheLogWriter(stdout if verbose else None), ),
(HandleRequestLog(),
(BasicHttpHandler(),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(static), ))),
(PathFilter('/', excluding=['/static']),
(DynamicHtml([dynamic],
reactor=reactor,
indexPage='/index'), )))))))
def testWriteLogMustNotFail(self):
logwriter = CallTrace('logwriter',
emptyGeneratorMethods=['someMessage'])
logwriter.exceptions['writeLog'] = ValueError
top = be((Observable(), (LogCollector('default'), (logwriter, ),
(FilterMessages(allowed=['someMessage']),
(LogKeyValue(dict(name='A')), )))))
with stderr_replaced() as err:
try:
consume(top.all.someMessage())
except ValueError:
self.fail("Should not raise an error; Only print it")
self.assertTrue('ValueError' in err.getvalue(), err.getvalue())
def testTransparentWontLogIfNothingPresentAll(self):
observer = createObserver()
observable = be((Observable(), (LogCollector(), (observer, ))))
result = asString(observable.all.allMessage('arg', kwarg='kwarg'))
self.assertEqual('allresult', result)
result = retval(observable.any.anyMessage('arg', kwarg='kwarg'))
self.assertEqual('anyresult', result)
observable.do.doMessage('arg', kwarg='kwarg')
result = observable.call.callMessage('arg', kwarg='kwarg')
self.assertEqual('callresult', result)
self.assertEqual(
['allMessage', 'anyMessage', 'doMessage', 'callMessage'],
observer.calledMethodNames())
for m in observer.calledMethods:
self.assertEqual(('arg', ), m.args)
self.assertEqual(dict(kwarg='kwarg'), m.kwargs)
def testEmptyQuery(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
requestHandler.returnValues['handleRequest'] = (f for f in [okXml, '<sru>', '</sru>'])
observable = be((Observable(),
(LogCollector(),
(self.handleRequestLog,
(requestHandler,)
),
(self.queryLogWriter,),
)
))
result = asString(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), arguments={}, path='/path/sru', otherKwarg='value'))
self.assertEquals(okXml+'<sru></sru>', result)
self.assertTrue(isfile(join(self.tempdir, '2009-11-02-query.log')))
self.assertEquals('2009-11-02T11:25:37Z 127.0.0.1 0.1K 1.000s - /path/sru \n', open(join(self.tempdir, '2009-11-02-query.log')).read())
def _buildDna(self, defaultRecordSchema="rdf", **kwargs):
jsonSearch = JsonSearch(defaultRecordSchema=defaultRecordSchema, enableCollectLog=True, **kwargs)
jsonSearch._timeNow = self._timeNow
class WriteLog:
def writeLog(mii, collectedLog):
self.logs.append(collectedLog)
self.dna = be(
(Observable(),
(LogCollector(),
(jsonSearch,
(self.observer, )
),
(WriteLog(),),
)
)
)
def testScope(self):
logwriter = CallTrace('logwriter',
emptyGeneratorMethods=['someMessage'])
top = be(
(Observable(), (LogCollector('default'), (logwriter, ),
(FilterMessages(allowed=['someMessage']), (
LogKeyValue(dict(name='A')),
(LogKeyValue(dict(name='B')), ),
(
LogCollectorScope('scope_one'),
(LogKeyValue(dict(name='C')), ),
),
(LogKeyValue(dict(name='D')), ),
(
LogCollectorScope('scope_two'),
(
LogKeyValue(dict(name='E')),
(
LogCollectorScope('scope_two_one'),
(LogKeyValue(dict(name='F')), ),
),
),
(LogKeyValue(dict(name='G')), ),
),
)))))
consume(top.all.someMessage())
self.assertEqual(['someMessage', 'writeLog'],
logwriter.calledMethodNames())
self.assertEqual(
{
'name': ['A', 'B', 'D'],
'scope_one': {
'name': ['C'],
},
'scope_two': {
'name': ['E', 'G'],
'scope_two_one': {
'name': ['F']
}
}
}, logwriter.calledMethods[-1].kwargs['collectedLog'])
def testAlmostNoData(self):
# No data due to HandleRequestLog is not used.
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
def handleRequest(**kwarg):
collectLog(dict(key='value'))
yield okXml
yield '<sru></sru>'
requestHandler.methods['handleRequest'] = handleRequest
observable = be((Observable(),
(LogCollector(),
(requestHandler,),
(self.queryLogWriter,),
)
))
result = asString(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), arguments={}, path='/path/sru', otherKwarg='value'))
self.assertEqual(okXml+'<sru></sru>', result)
self.assertEqual(0, len(listdir(self.tempdir)))
def testOne(self):
def fillReport(groups, collectedLog):
if not 'count' in groups:
groups['count'] = 0
groups['count'] += collectedLog['count'][0]
reports = []
def report(values):
reports.append(values.copy())
observer = CallTrace(methods={
'fillReport': fillReport,
'report': report
})
reactor = StoppableReactor()
class SomethingCounter(object):
def something(this):
collectLog(dict(count=1))
def reactorShutdown():
reactor.stop()
reactor.addTimer(2.5, reactorShutdown)
dna = be((Observable(), (LogCollector(),
(GustosContinuousLogWriter(reactor=reactor,
interval=1.0),
(observer, )), (SomethingCounter(), ))))
def job():
dna.do.something()
reactor.addTimer(0.5, job)
job()
reactor.loop()
self.assertEqual([
'analyseLog', 'fillReport', 'analyseLog', 'fillReport', 'report',
'analyseLog', 'fillReport', 'analyseLog', 'fillReport', 'report',
'analyseLog', 'fillReport'
], [m.name for m in observer.calledMethods])
self.assertEqual([{'count': 2}, {'count': 4}], reports)
def testLog(self):
class SomeLog(Transparent):
def logMe(self, argument):
collectLog(dict(logArgument=argument))
return self.call.logMe(argument=argument)
observer = createObserver()
observer.methods['doNotLogMe'] = observer.methods['logMe'] = observer.methods['callMessage']
observable = be((Observable(),
(LogCollector(),
(SomeLog(),
(observer,)
),
)
))
self.assertEquals('callresult', observable.call.logMe(argument=0))
self.assertEquals('callresult', observable.call.doNotLogMe(argument=0))
self.assertEquals(['logMe', 'writeLog', 'doNotLogMe'], observer.calledMethodNames())
writeLog = observer.calledMethods[1]
self.assertEquals((), writeLog.args)
self.assertEquals(['collectedLog'], writeLog.kwargs.keys())
self.assertEquals({'logArgument': [0]}, writeLog.kwargs['collectedLog'])
def testLogHttpError(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(),
(LogCollector(),
(handleRequestLog,
(requestHandler,),
),
(ApacheLogWriter(stream),),
)
))
# called by ObservableHttpServer
observable.do.logHttpError(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value')
logline = stream.getvalue()
self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 503 - "-" "-"\n', logline)
self.assertEqual(['logHttpError'], requestHandler.calledMethodNames())
self.assertEqual(dict(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value'), requestHandler.calledMethods[0].kwargs)
def testLoggedPathsNewStyle(self):
log = CallTrace('log')
def handleRequest(**kwargs):
yield okPlainText
yield 'result'
index = CallTrace('index', methods={'handleRequest': handleRequest})
observable = be((Observable(),
(LogCollector(),
(QueryLogWriter(log=log,
scopeNames=('global', 'yesPath')), ),
(LogCollectorScope('global'),
(HandleRequestLog(), (PathFilter('/yes'), (
LogCollectorScope('yesPath'),
(index, ),
)), (
PathFilter('/no'),
(index, ),
))))))
result = asString(
observable.all.handleRequest(Client=('11.22.33.44', 1234),
path='/yes'))
self.assertEquals(okPlainText + 'result', result)
result = asString(
observable.all.handleRequest(Client=('22.33.44.55', 2345),
path='/no'))
self.assertEquals(okPlainText + 'result', result)
result = asString(
observable.all.handleRequest(Client=('33.44.55.66', 3456),
path='/yes'))
self.assertEquals(okPlainText + 'result', result)
self.assertEquals(['log', 'log'], log.calledMethodNames())
self.assertEquals(['/yes', '/yes'],
[m.kwargs['path'] for m in log.calledMethods])
def testCollectLog(self):
__callstack_var_logCollector__ = LogCollector._logCollector()
collectLog(dict(key='value1', key2='value2'))
collectLog(dict(key='value3'))
self.assertEquals(dict(key=['value1', 'value3'], key2=['value2']), __callstack_var_logCollector__)
def dna(reactor, port, dataPath, logPath, statePath, externalUrl, customerLogoUrl, deproxyIps=None, **ignored):
environment = createEnvironment(dataPath)
harvesterData = environment.createHarvesterData()
harvesterDataRetrieve = environment.createHarvesterDataRetrieve()
deproxy = Deproxy(deproxyForIps=deproxyIps)
repositoryStatus = be(
(RepositoryStatus(logPath, statePath),
(harvesterData, )
)
)
configDict = JsonDict(
logPath=logPath,
statePath=statePath,
externaUrl=externalUrl,
dataPath=dataPath,
)
print("Started Metastreams with configuration:\n" + configDict.pretty_print())
userGroup = initializeUserGroupManagement(join(statePath, 'users'), harvesterData)
basicHtmlLoginHelix = (BasicHtmlLoginForm(
action="/login.action",
loginPath="/login",
home="/index",
rememberMeCookie=False,
lang="nl"),
(userGroup.basicHtmlObserver,),
)
varWwwdataPath = join(statePath, 'www-data', 'var')
isdir(varWwwdataPath) or makedirs(varWwwdataPath)
staticFilePaths = []
staticFiles = Transparent()
for path, libdir in [
('/js/bootstrap', '/usr/share/javascript/bootstrap5/js'),
('/css/bootstrap', '/usr/share/javascript/bootstrap5/css'),
('/css/bootstrap-icons', '/usr/share/javascript/bootstrap-icons'),
('/js/jquery', '/usr/share/javascript/jquery'),
('/js/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter'),
('/css/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter/css'),
('/js/autosize', '/usr/share/javascript/autosize'),
('/static', staticHtmlPath),
('/var', varWwwdataPath),
]:
staticFiles.addObserver(StaticFiles(libdir=libdir, path=path))
staticFilePaths.append(path)
return \
(Observable(),
(ObservableHttpServer(reactor, port),
(LogCollector(),
(ApacheLogWriter(stdout),),
(deproxy,
(HandleRequestLog(),
(BasicHttpHandler(),
(SessionHandler(),
(CookieMemoryStore(name="meresco-harvester", timeout=2*60*60), ),
(UserFromSession(),
(PathFilter("/info/version"),
(StringServer(VERSION_STRING, ContentTypePlainText), )
),
(PathFilter("/info/config"),
(StringServer(configDict.dumps(), ContentTypeJson), )
),
(PathFilter('/login.action'),
basicHtmlLoginHelix
),
(staticFiles,),
(PathFilter('/', excluding=['/info/version', '/info/config', '/action', '/login.action'] + harvesterDataRetrieve.paths + staticFilePaths),
(SecureZone("/login", excluding=["/index", "/invalid", "/rss", '/running.rss', '/showHarvesterStatus'], defaultLanguage="nl"),
(PathFilter('/', excluding=userGroup.excludedPaths),
(DynamicHtml(
[dynamicHtmlPath],
reactor=reactor,
additionalGlobals={
'externalUrl': externalUrl,
'escapeXml': escapeXml,
'compose': compose,
'dumps': dumps,
'VERSION': VERSION,
'CONFIG': configDict,
'Timeslot': Timeslot,
'ThroughputAnalyser': ThroughputAnalyser,
'dateSince': dateSince,
'callable': callable,
'OnlineHarvest': OnlineHarvest,
'StringIO': StringIO,
'okPlainText': okPlainText,
'ZuluTime': ZuluTime,
'xpathFirst': xpathFirst,
'customerLogoUrl': customerLogoUrl,
'uuid': lambda: str(uuid4()),
},
indexPage="/index",
),
basicHtmlLoginHelix,
(harvesterData,),
(repositoryStatus,),
(userGroup.dynamicHtmlObserver,),
)
),
(userGroup.actions,),
),
),
(PathFilter('/action'),
(HarvesterDataActions(),
(harvesterData,)
),
),
(PathFilter(harvesterDataRetrieve.paths),
(harvesterDataRetrieve,
(FilterFields(),
(harvesterData,),
),
(repositoryStatus,),
)
)
)
)
)
)
)
)
)
)