def testLogRequestInCaseOfExceptionAfterStatusCode(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
def handleRequestRaisesException(**kwargs):
yield Yield
yield okPlainText
yield 'text'
raise ValueError('doesntreallymatter')
yield
requestHandler.methods['handleRequest'] = handleRequestRaisesException
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(),
(LogCollector(),
(handleRequestLog,
(requestHandler,)
),
(ApacheLogWriter(stream),),
)
))
result = []
g = compose(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test'}, otherKwarg='value'))
try:
for x in g:
result.append(x)
except Exception:
pass
self.assertEqual([Yield, bytes(okPlainText, encoding='utf-8'), b'text'], result)
self.assertEqual(['handleRequest'], requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\')\n', logline)
def testApacheLog(self):
requestHandler = CallTrace(
'handler', ignoredAttributes=['writeLog', 'do_unknown'])
requestHandler.returnValues['handleRequest'] = (
f for f in [Yield, okPlainText, 'te', callable, 'xt'])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(), (
LogCollector(),
(handleRequestLog, (requestHandler, )),
(ApacheLogWriter(stream), ),
)))
result = asList(
observable.all.handleRequest(
Method='GET',
Client=('127.0.0.1', 1234),
RequestURI='http://example.org/path?key=value',
query='key=value',
path='/path',
Headers={
'Referer': 'http://meresco.org',
'User-Agent': 'Meresco-Components Test'
},
otherKwarg='value'))
self.assertEquals([Yield, okPlainText, 'te', callable, 'xt'], result)
self.assertEquals(['handleRequest'],
requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n',
logline)
def testApacheLog(self):
requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"])
requestHandler.returnValues["handleRequest"] = (f for f in [Yield, okPlainText, "te", callable, "xt"])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be(
(Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),)))
)
result = asList(
observable.all.handleRequest(
Method="GET",
Client=("127.0.0.1", 1234),
RequestURI="http://example.org/path?key=value",
query="key=value",
path="/path",
Headers={"Referer": "http://meresco.org", "User-Agent": "Meresco-Components Test"},
otherKwarg="value",
)
)
self.assertEquals([Yield, okPlainText, "te", callable, "xt"], result)
self.assertEquals(["handleRequest"], requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n',
logline,
)
def testAnalyseLog(self):
observer = CallTrace()
def handleRequest(**kwargs):
yield "We did it!"
handleRequestMock = CallTrace(methods={'handleRequest': handleRequest})
dna = be(
(Observable(),
(LogCollector(),
(GustosLogWriter(),
(AgentCountReport(gustosGroup="Gustos Group", scopeNames=('mock', )), ),
(observer, )
),
(LogCollectorScope("mock"),
(HandleRequestLog(),
(handleRequestMock, )
)
)
)
)
)
result = list(compose(dna.all.handleRequest(Headers={'User-Agent': GOOGLE_BOT})))
self.assertEqual(b"We did it!", result[0])
valuesKwarg = observer.calledMethods[-1].kwargs['values']
self.assertEqual({'Gustos Group': {'User agents': {'Googlebot/2.1': {'count': 1}}}}, valuesKwarg)
def setUp(self):
SeecrTestCase.setUp(self)
self.queryLogWriter = QueryLogWriter(DirectoryLog(self.tempdir))
self.handleRequestLog = HandleRequestLog()
self._timeNow = 1257161136.0 # 2009-11-02 11:30:00
def time():
self._timeNow += 1.0
return self._timeNow
self.handleRequestLog._time = time
def testLogRequestInCaseOfExceptionAfterStatusCode(self):
requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"])
def handleRequestRaisesException(**kwargs):
yield Yield
yield okPlainText
yield "text"
raise ValueError("doesntreallymatter")
yield
requestHandler.methods["handleRequest"] = handleRequestRaisesException
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be(
(Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),)))
)
result = []
g = compose(
observable.all.handleRequest(
Method="GET",
Client=("127.0.0.1", 1234),
RequestURI="http://example.org/path?key=value",
query="key=value",
path="/path",
Headers={"Referer": "http://meresco.org", "User-Agent": "Meresco-Components Test"},
otherKwarg="value",
)
)
try:
for x in g:
result.append(x)
except Exception:
pass
self.assertEquals([Yield, okPlainText, "text"], result)
self.assertEquals(["handleRequest"], requestHandler.calledMethodNames())
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\',)\n',
logline,
)
def testPostBody(self):
__callstack_var_logCollector__ = dict()
consume(HandleRequestLog().handleRequest(
Method='POST',
Client=('127.0.0.1', 1234),
RequestURI='http://example.org/path?key=value',
Headers={},
otherKwarg='value',
Body='short'))
self.assertEquals(
5, __callstack_var_logCollector__['httpRequest']['bodySize'][0])
def testDefaultTimeIsNow(self):
__callstack_var_logCollector__ = dict()
consume(HandleRequestLog().handleRequest(
Method='GET',
ResponseCode=503,
Client=('127.0.0.1', 1234),
RequestURI='http://example.org/path?key=value',
Headers={},
otherKwarg='value'))
self.assertAlmostEqual(
time(),
__callstack_var_logCollector__['httpRequest']['timestamp'][0],
places=1)
def dna(reactor, port, dynamic, static, verbose=True):
return (Observable(), (ObservableHttpServer(reactor, port=port),
(LogCollector(),
(ApacheLogWriter(stdout if verbose else None), ),
(HandleRequestLog(),
(BasicHttpHandler(),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(static), ))),
(PathFilter('/', excluding=['/static']),
(DynamicHtml([dynamic],
reactor=reactor,
indexPage='/index'), )))))))
def testLogHttpError(self):
requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown'])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be((Observable(),
(LogCollector(),
(handleRequestLog,
(requestHandler,),
),
(ApacheLogWriter(stream),),
)
))
# called by ObservableHttpServer
observable.do.logHttpError(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value')
logline = stream.getvalue()
self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 503 - "-" "-"\n', logline)
self.assertEqual(['logHttpError'], requestHandler.calledMethodNames())
self.assertEqual(dict(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value'), requestHandler.calledMethods[0].kwargs)
def testLogHttpError(self):
requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"])
stream = StringIO()
handleRequestLog = HandleRequestLog()
handleRequestLog._time = lambda: 1395409143.0
observable = be(
(Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),)))
)
# called by ObservableHttpServer
observable.do.logHttpError(
Method="GET",
ResponseCode=503,
Client=("127.0.0.1", 1234),
RequestURI="http://example.org/path?key=value",
Headers={},
otherKwarg="value",
)
logline = stream.getvalue()
self.assertEquals(
'127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 503 - "-" "-"\n', logline
)
self.assertEquals(["logHttpError"], requestHandler.calledMethodNames())
self.assertEquals(
dict(
Method="GET",
ResponseCode=503,
Client=("127.0.0.1", 1234),
RequestURI="http://example.org/path?key=value",
Headers={},
otherKwarg="value",
),
requestHandler.calledMethods[0].kwargs,
)
def testLoggedPathsNewStyle(self):
log = CallTrace('log')
def handleRequest(**kwargs):
yield okPlainText
yield 'result'
index = CallTrace('index', methods={'handleRequest': handleRequest})
observable = be((Observable(),
(LogCollector(),
(QueryLogWriter(log=log,
scopeNames=('global', 'yesPath')), ),
(LogCollectorScope('global'),
(HandleRequestLog(), (PathFilter('/yes'), (
LogCollectorScope('yesPath'),
(index, ),
)), (
PathFilter('/no'),
(index, ),
))))))
result = asString(
observable.all.handleRequest(Client=('11.22.33.44', 1234),
path='/yes'))
self.assertEquals(okPlainText + 'result', result)
result = asString(
observable.all.handleRequest(Client=('22.33.44.55', 2345),
path='/no'))
self.assertEquals(okPlainText + 'result', result)
result = asString(
observable.all.handleRequest(Client=('33.44.55.66', 3456),
path='/yes'))
self.assertEquals(okPlainText + 'result', result)
self.assertEquals(['log', 'log'], log.calledMethodNames())
self.assertEquals(['/yes', '/yes'],
[m.kwargs['path'] for m in log.calledMethods])
def dna(reactor, portNumber, config, tempDir, batchSize):
print 'Config', config
root = HandleRequestLog()
storage = DataStorage()
for data in config:
oaiName = ''.join(data['path'].split('/'))
oaiSuspendRegister = SuspendRegister()
try:
oaiJazz = OaiJazz(join(tempDir, oaiName), preciseDatestamp=True) # needed for backwards compatibility with meresco-oai versions preceding 5.16
except TypeError:
oaiJazz = OaiJazz(join(tempDir, oaiName))
oaiJazz = be(
(oaiJazz,
(oaiSuspendRegister,)
)
)
oaiJazzOperations = {
'ADD': oaiJazz.addOaiRecord,
'DEL': oaiJazz.deleteOaiRecord
}
for directory in data['dirs']:
for action, filename, setSpecs in iterOaiData(directory):
identifier, metadataPrefix = filename.rsplit('.', 1)
oaiJazzOperations[action](
identifier=identifier,
setSpecs=setSpecs,
metadataPrefixes=[metadataPrefix],
)
storage.addFile(filename, join(directory, escapeFilename(filename)))
sleep(0.000001)
oaiJazz.commit()
try:
oaiPmh = OaiPmh(repositoryName='Mock', adminEmail='*****@*****.**', supportXWait=True, batchSize=batchSize, preciseDatestamp=True)
except TypeError:
oaiPmh = OaiPmh(repositoryName='Mock', adminEmail='*****@*****.**', supportXWait=True, batchSize=batchSize) # needed for backwards compatibility with meresco-oai versions preceding 5.16
tree = be(
(PathFilter(data['path'], excluding=['/ready']),
(IllegalFromFix(),
(oaiPmh,
(oaiJazz,),
(oaiSuspendRegister,),
(storage,),
)
)
)
)
root.addObserver(tree)
return \
(Observable(),
(ObservableHttpServer(reactor, portNumber),
(LogCollector(),
(ApacheLogWriter(stdout),),
(root,
(PathFilter("/ready"),
(StringServer('yes', ContentTypePlainText),)
)
)
)
)
)
def dna(reactor, port, dataPath, logPath, statePath, externalUrl, customerLogoUrl, deproxyIps=None, **ignored):
environment = createEnvironment(dataPath)
harvesterData = environment.createHarvesterData()
harvesterDataRetrieve = environment.createHarvesterDataRetrieve()
deproxy = Deproxy(deproxyForIps=deproxyIps)
repositoryStatus = be(
(RepositoryStatus(logPath, statePath),
(harvesterData, )
)
)
configDict = JsonDict(
logPath=logPath,
statePath=statePath,
externaUrl=externalUrl,
dataPath=dataPath,
)
print("Started Metastreams with configuration:\n" + configDict.pretty_print())
userGroup = initializeUserGroupManagement(join(statePath, 'users'), harvesterData)
basicHtmlLoginHelix = (BasicHtmlLoginForm(
action="/login.action",
loginPath="/login",
home="/index",
rememberMeCookie=False,
lang="nl"),
(userGroup.basicHtmlObserver,),
)
varWwwdataPath = join(statePath, 'www-data', 'var')
isdir(varWwwdataPath) or makedirs(varWwwdataPath)
staticFilePaths = []
staticFiles = Transparent()
for path, libdir in [
('/js/bootstrap', '/usr/share/javascript/bootstrap5/js'),
('/css/bootstrap', '/usr/share/javascript/bootstrap5/css'),
('/css/bootstrap-icons', '/usr/share/javascript/bootstrap-icons'),
('/js/jquery', '/usr/share/javascript/jquery'),
('/js/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter'),
('/css/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter/css'),
('/js/autosize', '/usr/share/javascript/autosize'),
('/static', staticHtmlPath),
('/var', varWwwdataPath),
]:
staticFiles.addObserver(StaticFiles(libdir=libdir, path=path))
staticFilePaths.append(path)
return \
(Observable(),
(ObservableHttpServer(reactor, port),
(LogCollector(),
(ApacheLogWriter(stdout),),
(deproxy,
(HandleRequestLog(),
(BasicHttpHandler(),
(SessionHandler(),
(CookieMemoryStore(name="meresco-harvester", timeout=2*60*60), ),
(UserFromSession(),
(PathFilter("/info/version"),
(StringServer(VERSION_STRING, ContentTypePlainText), )
),
(PathFilter("/info/config"),
(StringServer(configDict.dumps(), ContentTypeJson), )
),
(PathFilter('/login.action'),
basicHtmlLoginHelix
),
(staticFiles,),
(PathFilter('/', excluding=['/info/version', '/info/config', '/action', '/login.action'] + harvesterDataRetrieve.paths + staticFilePaths),
(SecureZone("/login", excluding=["/index", "/invalid", "/rss", '/running.rss', '/showHarvesterStatus'], defaultLanguage="nl"),
(PathFilter('/', excluding=userGroup.excludedPaths),
(DynamicHtml(
[dynamicHtmlPath],
reactor=reactor,
additionalGlobals={
'externalUrl': externalUrl,
'escapeXml': escapeXml,
'compose': compose,
'dumps': dumps,
'VERSION': VERSION,
'CONFIG': configDict,
'Timeslot': Timeslot,
'ThroughputAnalyser': ThroughputAnalyser,
'dateSince': dateSince,
'callable': callable,
'OnlineHarvest': OnlineHarvest,
'StringIO': StringIO,
'okPlainText': okPlainText,
'ZuluTime': ZuluTime,
'xpathFirst': xpathFirst,
'customerLogoUrl': customerLogoUrl,
'uuid': lambda: str(uuid4()),
},
indexPage="/index",
),
basicHtmlLoginHelix,
(harvesterData,),
(repositoryStatus,),
(userGroup.dynamicHtmlObserver,),
)
),
(userGroup.actions,),
),
),
(PathFilter('/action'),
(HarvesterDataActions(),
(harvesterData,)
),
),
(PathFilter(harvesterDataRetrieve.paths),
(harvesterDataRetrieve,
(FilterFields(),
(harvesterData,),
),
(repositoryStatus,),
)
)
)
)
)
)
)
)
)
)
