def testLogRequestInCaseOfExceptionAfterStatusCode(self): requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown']) def handleRequestRaisesException(**kwargs): yield Yield yield okPlainText yield 'text' raise ValueError('doesntreallymatter') yield requestHandler.methods['handleRequest'] = handleRequestRaisesException stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), (LogCollector(), (handleRequestLog, (requestHandler,) ), (ApacheLogWriter(stream),), ) )) result = [] g = compose(observable.all.handleRequest(Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test'}, otherKwarg='value')) try: for x in g: result.append(x) except Exception: pass self.assertEqual([Yield, bytes(okPlainText, encoding='utf-8'), b'text'], result) self.assertEqual(['handleRequest'], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\')\n', logline)
def testApacheLog(self): requestHandler = CallTrace( 'handler', ignoredAttributes=['writeLog', 'do_unknown']) requestHandler.returnValues['handleRequest'] = ( f for f in [Yield, okPlainText, 'te', callable, 'xt']) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), ( LogCollector(), (handleRequestLog, (requestHandler, )), (ApacheLogWriter(stream), ), ))) result = asList( observable.all.handleRequest( Method='GET', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', query='key=value', path='/path', Headers={ 'Referer': 'http://meresco.org', 'User-Agent': 'Meresco-Components Test' }, otherKwarg='value')) self.assertEquals([Yield, okPlainText, 'te', callable, 'xt'], result) self.assertEquals(['handleRequest'], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n', logline)
def testApacheLog(self): requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"]) requestHandler.returnValues["handleRequest"] = (f for f in [Yield, okPlainText, "te", callable, "xt"]) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be( (Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),))) ) result = asList( observable.all.handleRequest( Method="GET", Client=("127.0.0.1", 1234), RequestURI="http://example.org/path?key=value", query="key=value", path="/path", Headers={"Referer": "http://meresco.org", "User-Agent": "Meresco-Components Test"}, otherKwarg="value", ) ) self.assertEquals([Yield, okPlainText, "te", callable, "xt"], result) self.assertEquals(["handleRequest"], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test"\n', logline, )
def testAnalyseLog(self): observer = CallTrace() def handleRequest(**kwargs): yield "We did it!" handleRequestMock = CallTrace(methods={'handleRequest': handleRequest}) dna = be( (Observable(), (LogCollector(), (GustosLogWriter(), (AgentCountReport(gustosGroup="Gustos Group", scopeNames=('mock', )), ), (observer, ) ), (LogCollectorScope("mock"), (HandleRequestLog(), (handleRequestMock, ) ) ) ) ) ) result = list(compose(dna.all.handleRequest(Headers={'User-Agent': GOOGLE_BOT}))) self.assertEqual(b"We did it!", result[0]) valuesKwarg = observer.calledMethods[-1].kwargs['values'] self.assertEqual({'Gustos Group': {'User agents': {'Googlebot/2.1': {'count': 1}}}}, valuesKwarg)
def setUp(self): SeecrTestCase.setUp(self) self.queryLogWriter = QueryLogWriter(DirectoryLog(self.tempdir)) self.handleRequestLog = HandleRequestLog() self._timeNow = 1257161136.0 # 2009-11-02 11:30:00 def time(): self._timeNow += 1.0 return self._timeNow self.handleRequestLog._time = time
def testLogRequestInCaseOfExceptionAfterStatusCode(self): requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"]) def handleRequestRaisesException(**kwargs): yield Yield yield okPlainText yield "text" raise ValueError("doesntreallymatter") yield requestHandler.methods["handleRequest"] = handleRequestRaisesException stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be( (Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),))) ) result = [] g = compose( observable.all.handleRequest( Method="GET", Client=("127.0.0.1", 1234), RequestURI="http://example.org/path?key=value", query="key=value", path="/path", Headers={"Referer": "http://meresco.org", "User-Agent": "Meresco-Components Test"}, otherKwarg="value", ) ) try: for x in g: result.append(x) except Exception: pass self.assertEquals([Yield, okPlainText, "text"], result) self.assertEquals(["handleRequest"], requestHandler.calledMethodNames()) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 200 64 "http://meresco.org" "Meresco-Components Test" Exception raised:\n ValueError(\'doesntreallymatter\',)\n', logline, )
def testPostBody(self): __callstack_var_logCollector__ = dict() consume(HandleRequestLog().handleRequest( Method='POST', Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value', Body='short')) self.assertEquals( 5, __callstack_var_logCollector__['httpRequest']['bodySize'][0])
def testDefaultTimeIsNow(self): __callstack_var_logCollector__ = dict() consume(HandleRequestLog().handleRequest( Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value')) self.assertAlmostEqual( time(), __callstack_var_logCollector__['httpRequest']['timestamp'][0], places=1)
def dna(reactor, port, dynamic, static, verbose=True): return (Observable(), (ObservableHttpServer(reactor, port=port), (LogCollector(), (ApacheLogWriter(stdout if verbose else None), ), (HandleRequestLog(), (BasicHttpHandler(), (PathFilter('/static'), (PathRename(lambda path: path[len('/static'):]), (FileServer(static), ))), (PathFilter('/', excluding=['/static']), (DynamicHtml([dynamic], reactor=reactor, indexPage='/index'), )))))))
def testLogHttpError(self): requestHandler = CallTrace('handler', ignoredAttributes=['writeLog', 'do_unknown']) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be((Observable(), (LogCollector(), (handleRequestLog, (requestHandler,), ), (ApacheLogWriter(stream),), ) )) # called by ObservableHttpServer observable.do.logHttpError(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value') logline = stream.getvalue() self.assertEqual('127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 503 - "-" "-"\n', logline) self.assertEqual(['logHttpError'], requestHandler.calledMethodNames()) self.assertEqual(dict(Method='GET', ResponseCode=503, Client=('127.0.0.1', 1234), RequestURI='http://example.org/path?key=value', Headers={}, otherKwarg='value'), requestHandler.calledMethods[0].kwargs)
def testLogHttpError(self): requestHandler = CallTrace("handler", ignoredAttributes=["writeLog", "do_unknown"]) stream = StringIO() handleRequestLog = HandleRequestLog() handleRequestLog._time = lambda: 1395409143.0 observable = be( (Observable(), (LogCollector(), (handleRequestLog, (requestHandler,)), (ApacheLogWriter(stream),))) ) # called by ObservableHttpServer observable.do.logHttpError( Method="GET", ResponseCode=503, Client=("127.0.0.1", 1234), RequestURI="http://example.org/path?key=value", Headers={}, otherKwarg="value", ) logline = stream.getvalue() self.assertEquals( '127.0.0.1 - - [21/Mar/2014:13:39:03 +0000] "GET /path?key=value HTTP/1.0" 503 - "-" "-"\n', logline ) self.assertEquals(["logHttpError"], requestHandler.calledMethodNames()) self.assertEquals( dict( Method="GET", ResponseCode=503, Client=("127.0.0.1", 1234), RequestURI="http://example.org/path?key=value", Headers={}, otherKwarg="value", ), requestHandler.calledMethods[0].kwargs, )
def testLoggedPathsNewStyle(self): log = CallTrace('log') def handleRequest(**kwargs): yield okPlainText yield 'result' index = CallTrace('index', methods={'handleRequest': handleRequest}) observable = be((Observable(), (LogCollector(), (QueryLogWriter(log=log, scopeNames=('global', 'yesPath')), ), (LogCollectorScope('global'), (HandleRequestLog(), (PathFilter('/yes'), ( LogCollectorScope('yesPath'), (index, ), )), ( PathFilter('/no'), (index, ), )))))) result = asString( observable.all.handleRequest(Client=('11.22.33.44', 1234), path='/yes')) self.assertEquals(okPlainText + 'result', result) result = asString( observable.all.handleRequest(Client=('22.33.44.55', 2345), path='/no')) self.assertEquals(okPlainText + 'result', result) result = asString( observable.all.handleRequest(Client=('33.44.55.66', 3456), path='/yes')) self.assertEquals(okPlainText + 'result', result) self.assertEquals(['log', 'log'], log.calledMethodNames()) self.assertEquals(['/yes', '/yes'], [m.kwargs['path'] for m in log.calledMethods])
def dna(reactor, portNumber, config, tempDir, batchSize): print 'Config', config root = HandleRequestLog() storage = DataStorage() for data in config: oaiName = ''.join(data['path'].split('/')) oaiSuspendRegister = SuspendRegister() try: oaiJazz = OaiJazz(join(tempDir, oaiName), preciseDatestamp=True) # needed for backwards compatibility with meresco-oai versions preceding 5.16 except TypeError: oaiJazz = OaiJazz(join(tempDir, oaiName)) oaiJazz = be( (oaiJazz, (oaiSuspendRegister,) ) ) oaiJazzOperations = { 'ADD': oaiJazz.addOaiRecord, 'DEL': oaiJazz.deleteOaiRecord } for directory in data['dirs']: for action, filename, setSpecs in iterOaiData(directory): identifier, metadataPrefix = filename.rsplit('.', 1) oaiJazzOperations[action]( identifier=identifier, setSpecs=setSpecs, metadataPrefixes=[metadataPrefix], ) storage.addFile(filename, join(directory, escapeFilename(filename))) sleep(0.000001) oaiJazz.commit() try: oaiPmh = OaiPmh(repositoryName='Mock', adminEmail='*****@*****.**', supportXWait=True, batchSize=batchSize, preciseDatestamp=True) except TypeError: oaiPmh = OaiPmh(repositoryName='Mock', adminEmail='*****@*****.**', supportXWait=True, batchSize=batchSize) # needed for backwards compatibility with meresco-oai versions preceding 5.16 tree = be( (PathFilter(data['path'], excluding=['/ready']), (IllegalFromFix(), (oaiPmh, (oaiJazz,), (oaiSuspendRegister,), (storage,), ) ) ) ) root.addObserver(tree) return \ (Observable(), (ObservableHttpServer(reactor, portNumber), (LogCollector(), (ApacheLogWriter(stdout),), (root, (PathFilter("/ready"), (StringServer('yes', ContentTypePlainText),) ) ) ) ) )
def dna(reactor, port, dataPath, logPath, statePath, externalUrl, customerLogoUrl, deproxyIps=None, **ignored): environment = createEnvironment(dataPath) harvesterData = environment.createHarvesterData() harvesterDataRetrieve = environment.createHarvesterDataRetrieve() deproxy = Deproxy(deproxyForIps=deproxyIps) repositoryStatus = be( (RepositoryStatus(logPath, statePath), (harvesterData, ) ) ) configDict = JsonDict( logPath=logPath, statePath=statePath, externaUrl=externalUrl, dataPath=dataPath, ) print("Started Metastreams with configuration:\n" + configDict.pretty_print()) userGroup = initializeUserGroupManagement(join(statePath, 'users'), harvesterData) basicHtmlLoginHelix = (BasicHtmlLoginForm( action="/login.action", loginPath="/login", home="/index", rememberMeCookie=False, lang="nl"), (userGroup.basicHtmlObserver,), ) varWwwdataPath = join(statePath, 'www-data', 'var') isdir(varWwwdataPath) or makedirs(varWwwdataPath) staticFilePaths = [] staticFiles = Transparent() for path, libdir in [ ('/js/bootstrap', '/usr/share/javascript/bootstrap5/js'), ('/css/bootstrap', '/usr/share/javascript/bootstrap5/css'), ('/css/bootstrap-icons', '/usr/share/javascript/bootstrap-icons'), ('/js/jquery', '/usr/share/javascript/jquery'), ('/js/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter'), ('/css/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter/css'), ('/js/autosize', '/usr/share/javascript/autosize'), ('/static', staticHtmlPath), ('/var', varWwwdataPath), ]: staticFiles.addObserver(StaticFiles(libdir=libdir, path=path)) staticFilePaths.append(path) return \ (Observable(), (ObservableHttpServer(reactor, port), (LogCollector(), (ApacheLogWriter(stdout),), (deproxy, (HandleRequestLog(), (BasicHttpHandler(), (SessionHandler(), (CookieMemoryStore(name="meresco-harvester", timeout=2*60*60), ), (UserFromSession(), (PathFilter("/info/version"), (StringServer(VERSION_STRING, ContentTypePlainText), ) ), (PathFilter("/info/config"), (StringServer(configDict.dumps(), ContentTypeJson), ) ), (PathFilter('/login.action'), basicHtmlLoginHelix ), (staticFiles,), (PathFilter('/', excluding=['/info/version', '/info/config', '/action', '/login.action'] + harvesterDataRetrieve.paths + staticFilePaths), (SecureZone("/login", excluding=["/index", "/invalid", "/rss", '/running.rss', '/showHarvesterStatus'], defaultLanguage="nl"), (PathFilter('/', excluding=userGroup.excludedPaths), (DynamicHtml( [dynamicHtmlPath], reactor=reactor, additionalGlobals={ 'externalUrl': externalUrl, 'escapeXml': escapeXml, 'compose': compose, 'dumps': dumps, 'VERSION': VERSION, 'CONFIG': configDict, 'Timeslot': Timeslot, 'ThroughputAnalyser': ThroughputAnalyser, 'dateSince': dateSince, 'callable': callable, 'OnlineHarvest': OnlineHarvest, 'StringIO': StringIO, 'okPlainText': okPlainText, 'ZuluTime': ZuluTime, 'xpathFirst': xpathFirst, 'customerLogoUrl': customerLogoUrl, 'uuid': lambda: str(uuid4()), }, indexPage="/index", ), basicHtmlLoginHelix, (harvesterData,), (repositoryStatus,), (userGroup.dynamicHtmlObserver,), ) ), (userGroup.actions,), ), ), (PathFilter('/action'), (HarvesterDataActions(), (harvesterData,) ), ), (PathFilter(harvesterDataRetrieve.paths), (harvesterDataRetrieve, (FilterFields(), (harvesterData,), ), (repositoryStatus,), ) ) ) ) ) ) ) ) ) )