def testApacheLog(self):
requestHandler = CallTrace(
'handler', ignoredAttributes=['writeLog', 'do_unknown'])
requestHandler.returnValues['handleRequest'] = (
f for f in [Yield, okPlainText, 'te', callable, 'xt'])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(), (
LogCollector(),
(handleRequestLog, (requestHandler, )),
(ApacheLogWriter(stream), ),
)))
result = asList(
observable.all.handleRequest(
Method='GET',
Client=('127.0.0.1', 1234),
RequestURI='http://example.org/path?key=value',
query='key=value',
path='/path',
Headers={
'Referer': 'http://meresco.org',
'User-Agent': 'Meresco-Components Test'
},
otherKwarg='value'))
self.assertEquals([Yield, okPlainText, 'te', callable, 'xt'], result)
self.assertEquals(['handleRequest'],
requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n',
logline)
def testLogRequestInCaseOfExceptionAfterStatusCode(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
def handleRequestRaisesException(**kwargs):
yield Yield
yield okPlainText
yield 'text'
raise ValueError('doesntreallymatter')
yield
requestHandler.methods['handleRequest'] = handleRequestRaisesException
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(),
(LogCollector(),
(handleRequestLog,
(requestHandler,)
),
(ApacheLogWriter(stream),),
)
))
result = []
g = compose(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test'}, otherKwarg='value'))
try:
for x in g:
result.append(x)
except Exception:
pass
self.assertEqual([Yield, bytes(okPlainText, encoding='utf-8'), b'text'], result)
self.assertEqual(['handleRequest'], requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\')\n', logline)
def testNormalLog(self):
stream = StringIO()
writer = ApacheLogWriter(stream)
writer.writeLog(
collectedLog={
'httpRequest': {
'Headers': [{}],
'Client': [('10.11.12.13', 12345)],
'timestamp': [1468845136.824253],
'Method': ['GET'],
'RequestURI': ['/abc'],
},
'httpResponse': {
'httpStatus': ['200'],
}
}
)
self.assertEquals('10.11.12.13 - - [18/Jul/2016:12:32:16 +0000] "GET /abc HTTP/1.0" 200 - "-" "-"\n', stream.getvalue())
def testNormalLog(self):
stream = StringIO()
writer = ApacheLogWriter(stream)
writer.writeLog(
collectedLog={
'httpRequest': {
'Headers': [{}],
'Client': [('10.11.12.13', 12345)],
'timestamp': [1468845136.824253],
'Method': ['GET'],
'RequestURI': ['/abc'],
},
'httpResponse': {
'httpStatus': ['200'],
}
})
self.assertEquals(
'10.11.12.13 - - [18/Jul/2016:12:32:16 +0000] "GET /abc HTTP/1.0" 200 - "-" "-"\n',
stream.getvalue())
def testLogWithException(self):
stream = StringIO()
writer = ApacheLogWriter(stream)
writer.writeLog(
collectedLog={
'httpRequest': {
'Headers': [{}],
'Client': [('10.11.12.13', 12345)],
'timestamp': [1468845136.824253],
'Method': ['GET'],
'RequestURI': ['/abc'],
},
'httpResponse': {
'httpStatus': ['200'],
'exception': [ValueError('xyz')]
}
})
self.assertEqual(
'10.11.12.13 - - [18/Jul/2016:12:32:16 +0000] "GET /abc HTTP/1.0" 200 - "-" "-" Exception raised:\n ValueError(\'xyz\')\n',
stream.getvalue())
def testRejectLog(self):
stream = StringIO()
writer = ApacheLogWriter(stream, rejectLog=lambda collectedLog: getScoped(collectedLog, scopeNames=(), key='httpRequest').get('RequestURI') == ['/abc'])
collectedLog={
'httpRequest': {
'Headers': [{}],
'Client': [('10.11.12.13', 12345)],
'timestamp': [1468845136.824253],
'Method': ['GET'],
'RequestURI': ['/abc'],
},
'httpResponse': {
'httpStatus': ['200'],
}
}
writer.writeLog(collectedLog)
self.assertEquals('', stream.getvalue())
collectedLog['httpRequest']['RequestURI'] = ['/abcd']
writer.writeLog(collectedLog)
self.assertEquals('10.11.12.13 - - [18/Jul/2016:12:32:16 +0000] "GET /abcd HTTP/1.0" 200 - "-" "-"\n', stream.getvalue())
def testRejectLog(self):
stream = StringIO()
writer = ApacheLogWriter(
stream,
rejectLog=lambda collectedLog: getScoped(
collectedLog, scopeNames=(), key='httpRequest').get(
'RequestURI') == ['/abc'])
collectedLog = {
'httpRequest': {
'Headers': [{}],
'Client': [('10.11.12.13', 12345)],
'timestamp': [1468845136.824253],
'Method': ['GET'],
'RequestURI': ['/abc'],
},
'httpResponse': {
'httpStatus': ['200'],
}
}
writer.writeLog(collectedLog)
self.assertEquals('', stream.getvalue())
collectedLog['httpRequest']['RequestURI'] = ['/abcd']
writer.writeLog(collectedLog)
self.assertEquals(
'10.11.12.13 - - [18/Jul/2016:12:32:16 +0000] "GET /abcd HTTP/1.0" 200 - "-" "-"\n',
stream.getvalue())
def dna(reactor, port, dynamic, static, verbose=True):
return (Observable(), (ObservableHttpServer(reactor, port=port),
(LogCollector(),
(ApacheLogWriter(stdout if verbose else None), ),
(HandleRequestLog(),
(BasicHttpHandler(),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(static), ))),
(PathFilter('/', excluding=['/static']),
(DynamicHtml([dynamic],
reactor=reactor,
indexPage='/index'), )))))))
def testLogHttpError(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(),
(LogCollector(),
(handleRequestLog,
(requestHandler,),
),
(ApacheLogWriter(stream),),
)
))
# called by ObservableHttpServer
observable.do.logHttpError(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value')
logline = stream.getvalue()
self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 503 - "-" "-"\n', logline)
self.assertEqual(['logHttpError'], requestHandler.calledMethodNames())
self.assertEqual(dict(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value'), requestHandler.calledMethods[0].kwargs)
def testWriteLogWithoutSensibleData(self):
stream = StringIO()
writer = ApacheLogWriter(stream)
writer.writeLog(collectedLog={'key':['value']})
self.assertEquals("", stream.getvalue())
def testWriteLogWithoutSensibleData(self):
stream = StringIO()
writer = ApacheLogWriter(stream)
writer.writeLog(collectedLog={'key': ['value']})
self.assertEquals("", stream.getvalue())
def dna(reactor, port, dataPath, logPath, statePath, externalUrl, customerLogoUrl, deproxyIps=None, **ignored):
environment = createEnvironment(dataPath)
harvesterData = environment.createHarvesterData()
harvesterDataRetrieve = environment.createHarvesterDataRetrieve()
deproxy = Deproxy(deproxyForIps=deproxyIps)
repositoryStatus = be(
(RepositoryStatus(logPath, statePath),
(harvesterData, )
)
)
configDict = JsonDict(
logPath=logPath,
statePath=statePath,
externaUrl=externalUrl,
dataPath=dataPath,
)
print("Started Metastreams with configuration:\n" + configDict.pretty_print())
userGroup = initializeUserGroupManagement(join(statePath, 'users'), harvesterData)
basicHtmlLoginHelix = (BasicHtmlLoginForm(
action="/login.action",
loginPath="/login",
home="/index",
rememberMeCookie=False,
lang="nl"),
(userGroup.basicHtmlObserver,),
)
varWwwdataPath = join(statePath, 'www-data', 'var')
isdir(varWwwdataPath) or makedirs(varWwwdataPath)
staticFilePaths = []
staticFiles = Transparent()
for path, libdir in [
('/js/bootstrap', '/usr/share/javascript/bootstrap5/js'),
('/css/bootstrap', '/usr/share/javascript/bootstrap5/css'),
('/css/bootstrap-icons', '/usr/share/javascript/bootstrap-icons'),
('/js/jquery', '/usr/share/javascript/jquery'),
('/js/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter'),
('/css/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter/css'),
('/js/autosize', '/usr/share/javascript/autosize'),
('/static', staticHtmlPath),
('/var', varWwwdataPath),
]:
staticFiles.addObserver(StaticFiles(libdir=libdir, path=path))
staticFilePaths.append(path)
return \
(Observable(),
(ObservableHttpServer(reactor, port),
(LogCollector(),
(ApacheLogWriter(stdout),),
(deproxy,
(HandleRequestLog(),
(BasicHttpHandler(),
(SessionHandler(),
(CookieMemoryStore(name="meresco-harvester", timeout=2*60*60), ),
(UserFromSession(),
(PathFilter("/info/version"),
(StringServer(VERSION_STRING, ContentTypePlainText), )
),
(PathFilter("/info/config"),
(StringServer(configDict.dumps(), ContentTypeJson), )
),
(PathFilter('/login.action'),
basicHtmlLoginHelix
),
(staticFiles,),
(PathFilter('/', excluding=['/info/version', '/info/config', '/action', '/login.action'] + harvesterDataRetrieve.paths + staticFilePaths),
(SecureZone("/login", excluding=["/index", "/invalid", "/rss", '/running.rss', '/showHarvesterStatus'], defaultLanguage="nl"),
(PathFilter('/', excluding=userGroup.excludedPaths),
(DynamicHtml(
[dynamicHtmlPath],
reactor=reactor,
additionalGlobals={
'externalUrl': externalUrl,
'escapeXml': escapeXml,
'compose': compose,
'dumps': dumps,
'VERSION': VERSION,
'CONFIG': configDict,
'Timeslot': Timeslot,
'ThroughputAnalyser': ThroughputAnalyser,
'dateSince': dateSince,
'callable': callable,
'OnlineHarvest': OnlineHarvest,
'StringIO': StringIO,
'okPlainText': okPlainText,
'ZuluTime': ZuluTime,
'xpathFirst': xpathFirst,
'customerLogoUrl': customerLogoUrl,
'uuid': lambda: str(uuid4()),
},
indexPage="/index",
),
basicHtmlLoginHelix,
(harvesterData,),
(repositoryStatus,),
(userGroup.dynamicHtmlObserver,),
)
),
(userGroup.actions,),
),
),
(PathFilter('/action'),
(HarvesterDataActions(),
(harvesterData,)
),
),
(PathFilter(harvesterDataRetrieve.paths),
(harvesterDataRetrieve,
(FilterFields(),
(harvesterData,),
),
(repositoryStatus,),
)
)
)
)
)
)
)
)
)
)
