def createTree(self, **kwargs):
self.deproxy = Deproxy(**kwargs)
self.observer = CallTrace('Observer', emptyGeneratorMethods=['handleRequest'])
self.top = be((Observable(),
(self.deproxy,
(self.observer,),
)
))
def createTree(self, **kwargs):
self.deproxy = Deproxy(**kwargs)
self.observer = CallTrace('Observer', emptyGeneratorMethods=['handleRequest'])
self.top = be((Observable(),
(self.deproxy,
(self.observer,),
)
))
class DeproxyTest(TestCase):
def setUp(self):
TestCase.setUp(self)
self.createTree(deproxyForIps=['1.1.1.1'])
def createTree(self, **kwargs):
self.deproxy = Deproxy(**kwargs)
self.observer = CallTrace('Observer', emptyGeneratorMethods=['handleRequest'])
self.top = be((Observable(),
(self.deproxy,
(self.observer,),
)
))
def testShouldPassthroughHandleRequestIfUnconfigured(self):
self.createTree()
consume(self.top.all.handleRequest(Client=("9.1.8.2", 99), Headers={'H': 'eaders'}, other='item'))
self.assertEquals(['handleRequest'], self.observer.calledMethodNames())
handleRequest, = self.observer.calledMethods
self.assertEquals(tuple(), handleRequest.args)
self.assertEquals(dict(
Client=("9.1.8.2", 99),
Headers={'H': 'eaders'},
port=80,
other='item',
OriginalClient=None,
), handleRequest.kwargs)
def testClientInCaseNoXForwardedForHeader(self):
list(compose(self.top.all.handleRequest(Client=("1.1.1.1", 11111), Headers={})))
self.assertEquals(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEquals("1.1.1.1", handleRequestCallKwargs['Client'][0])
self.assertEquals({}, handleRequestCallKwargs['Headers'])
def testDeproxy(self):
list(compose(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": "2.2.2.2"})))
self.assertEquals(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEquals("2.2.2.2", handleRequestCallKwargs['Client'][0])
self.assertEquals("1.1.1.1", handleRequestCallKwargs['OriginalClient'][0])
self.assertEquals({"X-Forwarded-For": "2.2.2.2"}, handleRequestCallKwargs['Headers'])
def testClientFromMulitpleXForwardedForEntries(self):
list(compose(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": "2.2.2.2,3.3.3.3,4.4.4.4"})))
self.assertEquals(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEquals('4.4.4.4', handleRequestCallKwargs['Client'][0])
self.assertEquals({"X-Forwarded-For": "2.2.2.2,3.3.3.3,4.4.4.4"}, handleRequestCallKwargs['Headers'])
list(compose(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": " 2.2.2.2 , 3.3.3.3, 4.4.4.4 ,"})))
self.assertEquals('4.4.4.4', self.observer.calledMethods[1].kwargs['Client'][0])
def testHostFromXForwardedHost(self):
self.createTree(deproxyForIpRanges=[
('9.9.9.0', '9.9.9.255')])
Headers={
"Host": "1.1.1.1:11111",
"X-Forwarded-Host": "2.2.2.2:22222,3.3.3.3:33333,4.4.4.4:44444"
}
consume(self.top.all.handleRequest(Client=("9.9.9.9", 9999), port=11111, Headers=Headers))
self.assertEquals(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEquals("4.4.4.4:44444", handleRequestCallKwargs['Headers']['Host'])
self.assertEquals(44444, handleRequestCallKwargs['port'])
Headers={
"Host": "1.1.1.1:11111",
"X-Forwarded-Host": "2.2.2.2,3.3.3.3,4.4.4.4"
}
consume(self.top.all.handleRequest(Client=("9.9.9.9", 9999), port=11111, Headers=Headers))
self.assertEquals(2, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[1].kwargs
self.assertEquals('4.4.4.4', handleRequestCallKwargs['Headers']['Host'])
self.assertEquals(80, handleRequestCallKwargs['port'])
def testDeproxyForIps(self):
self.createTree(deproxyForIps=['3.3.3.3'])
consume(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": "2.2.2.2"}))
self.assertEquals(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEquals("1.1.1.1", handleRequestCallKwargs['Client'][0])
self.assertEquals({"X-Forwarded-For": "2.2.2.2"}, handleRequestCallKwargs['Headers'])
def testDeproxyUpdateIps(self):
# Expose updateIps from IpFilter
self.createTree(deproxyForIps=['127.7.7.7'])
# White box
allowDeproxying = lambda ip: self.deproxy._ipfilter.filterIpAddress(ip)
self.assertEquals(True, allowDeproxying('127.7.7.7'))
self.assertEquals(False, allowDeproxying('127.0.0.1'))
self.assertEquals(False, allowDeproxying('10.0.0.1'))
self.deproxy.updateIps(ipAddresses=['192.168.96.96'], ipRanges=[('10.0.0.0', '10.0.0.2')])
self.assertEquals(True, allowDeproxying('192.168.96.96'))
self.assertEquals(True, allowDeproxying('10.0.0.1'))
self.assertEquals(False, allowDeproxying('127.7.7.7'))
self.assertEquals(False, allowDeproxying('127.0.0.1'))
# Black box
consume(self.top.all.handleRequest(
Client=("192.168.96.96", 12345),
Headers={
"X-Forwarded-For": "2.2.2.2",
"X-Forwarded-Host": "example.org"}))
self.assertEquals(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEquals("2.2.2.2", handleRequestCallKwargs['Client'][0])
self.assertEquals('192.168.96.96', handleRequestCallKwargs['OriginalClient'][0])
self.assertEquals({"X-Forwarded-For": "2.2.2.2", "X-Forwarded-Host": "example.org", "Host": "example.org"}, handleRequestCallKwargs['Headers'])
def testOnlyDeproxied(self):
odp = OnlyDeproxied()
odp.addObserver(self.observer)
consume(odp.handleRequest(path='/path'))
self.assertEquals([], self.observer.calledMethodNames())
consume(odp.handleRequest(path='/path', OriginalClient=('1.2.3.4', 1234)))
self.assertEquals(['handleRequest'], self.observer.calledMethodNames())
self.assertEquals(dict(path='/path', OriginalClient=('1.2.3.4', 1234)), self.observer.calledMethods[0].kwargs)
def dna(reactor, port, dataPath, logPath, statePath, externalUrl, customerLogoUrl, deproxyIps=None, **ignored):
environment = createEnvironment(dataPath)
harvesterData = environment.createHarvesterData()
harvesterDataRetrieve = environment.createHarvesterDataRetrieve()
deproxy = Deproxy(deproxyForIps=deproxyIps)
repositoryStatus = be(
(RepositoryStatus(logPath, statePath),
(harvesterData, )
)
)
configDict = JsonDict(
logPath=logPath,
statePath=statePath,
externaUrl=externalUrl,
dataPath=dataPath,
)
print("Started Metastreams with configuration:\n" + configDict.pretty_print())
userGroup = initializeUserGroupManagement(join(statePath, 'users'), harvesterData)
basicHtmlLoginHelix = (BasicHtmlLoginForm(
action="/login.action",
loginPath="/login",
home="/index",
rememberMeCookie=False,
lang="nl"),
(userGroup.basicHtmlObserver,),
)
varWwwdataPath = join(statePath, 'www-data', 'var')
isdir(varWwwdataPath) or makedirs(varWwwdataPath)
staticFilePaths = []
staticFiles = Transparent()
for path, libdir in [
('/js/bootstrap', '/usr/share/javascript/bootstrap5/js'),
('/css/bootstrap', '/usr/share/javascript/bootstrap5/css'),
('/css/bootstrap-icons', '/usr/share/javascript/bootstrap-icons'),
('/js/jquery', '/usr/share/javascript/jquery'),
('/js/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter'),
('/css/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter/css'),
('/js/autosize', '/usr/share/javascript/autosize'),
('/static', staticHtmlPath),
('/var', varWwwdataPath),
]:
staticFiles.addObserver(StaticFiles(libdir=libdir, path=path))
staticFilePaths.append(path)
return \
(Observable(),
(ObservableHttpServer(reactor, port),
(LogCollector(),
(ApacheLogWriter(stdout),),
(deproxy,
(HandleRequestLog(),
(BasicHttpHandler(),
(SessionHandler(),
(CookieMemoryStore(name="meresco-harvester", timeout=2*60*60), ),
(UserFromSession(),
(PathFilter("/info/version"),
(StringServer(VERSION_STRING, ContentTypePlainText), )
),
(PathFilter("/info/config"),
(StringServer(configDict.dumps(), ContentTypeJson), )
),
(PathFilter('/login.action'),
basicHtmlLoginHelix
),
(staticFiles,),
(PathFilter('/', excluding=['/info/version', '/info/config', '/action', '/login.action'] + harvesterDataRetrieve.paths + staticFilePaths),
(SecureZone("/login", excluding=["/index", "/invalid", "/rss", '/running.rss', '/showHarvesterStatus'], defaultLanguage="nl"),
(PathFilter('/', excluding=userGroup.excludedPaths),
(DynamicHtml(
[dynamicHtmlPath],
reactor=reactor,
additionalGlobals={
'externalUrl': externalUrl,
'escapeXml': escapeXml,
'compose': compose,
'dumps': dumps,
'VERSION': VERSION,
'CONFIG': configDict,
'Timeslot': Timeslot,
'ThroughputAnalyser': ThroughputAnalyser,
'dateSince': dateSince,
'callable': callable,
'OnlineHarvest': OnlineHarvest,
'StringIO': StringIO,
'okPlainText': okPlainText,
'ZuluTime': ZuluTime,
'xpathFirst': xpathFirst,
'customerLogoUrl': customerLogoUrl,
'uuid': lambda: str(uuid4()),
},
indexPage="/index",
),
basicHtmlLoginHelix,
(harvesterData,),
(repositoryStatus,),
(userGroup.dynamicHtmlObserver,),
)
),
(userGroup.actions,),
),
),
(PathFilter('/action'),
(HarvesterDataActions(),
(harvesterData,)
),
),
(PathFilter(harvesterDataRetrieve.paths),
(harvesterDataRetrieve,
(FilterFields(),
(harvesterData,),
),
(repositoryStatus,),
)
)
)
)
)
)
)
)
)
)
class DeproxyTest(TestCase):
def setUp(self):
TestCase.setUp(self)
self.createTree(deproxyForIps=['1.1.1.1'])
def createTree(self, **kwargs):
self.deproxy = Deproxy(**kwargs)
self.observer = CallTrace('Observer', emptyGeneratorMethods=['handleRequest'])
self.top = be((Observable(),
(self.deproxy,
(self.observer,),
)
))
def testShouldPassthroughHandleRequestIfUnconfigured(self):
self.createTree()
consume(self.top.all.handleRequest(Client=("9.1.8.2", 99), Headers={'H': 'eaders'}, other='item'))
self.assertEqual(['handleRequest'], self.observer.calledMethodNames())
handleRequest, = self.observer.calledMethods
self.assertEqual(tuple(), handleRequest.args)
self.assertEqual(dict(
Client=("9.1.8.2", 99),
Headers={'H': 'eaders'},
port=80,
other='item',
OriginalClient=None,
), handleRequest.kwargs)
def testClientInCaseNoXForwardedForHeader(self):
list(compose(self.top.all.handleRequest(Client=("1.1.1.1", 11111), Headers={})))
self.assertEqual(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual("1.1.1.1", handleRequestCallKwargs['Client'][0])
self.assertEqual({}, handleRequestCallKwargs['Headers'])
def testDeproxy(self):
list(compose(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": "2.2.2.2"})))
self.assertEqual(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual("2.2.2.2", handleRequestCallKwargs['Client'][0])
self.assertEqual("1.1.1.1", handleRequestCallKwargs['OriginalClient'][0])
self.assertEqual({"X-Forwarded-For": "2.2.2.2"}, handleRequestCallKwargs['Headers'])
def testClientFromMulitpleXForwardedForEntries(self):
list(compose(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": "2.2.2.2,3.3.3.3,4.4.4.4"})))
self.assertEqual(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual('4.4.4.4', handleRequestCallKwargs['Client'][0])
self.assertEqual({"X-Forwarded-For": "2.2.2.2,3.3.3.3,4.4.4.4"}, handleRequestCallKwargs['Headers'])
list(compose(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": " 2.2.2.2 , 3.3.3.3, 4.4.4.4 ,"})))
self.assertEqual('4.4.4.4', self.observer.calledMethods[1].kwargs['Client'][0])
def testHostFromXForwardedHost(self):
self.createTree(deproxyForIpRanges=[
('9.9.9.0', '9.9.9.255')])
Headers={
"Host": "1.1.1.1:11111",
"X-Forwarded-Host": "2.2.2.2:22222,3.3.3.3:33333,4.4.4.4:44444"
}
consume(self.top.all.handleRequest(Client=("9.9.9.9", 9999), port=11111, Headers=Headers))
self.assertEqual(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual("4.4.4.4:44444", handleRequestCallKwargs['Headers']['Host'])
self.assertEqual(44444, handleRequestCallKwargs['port'])
Headers={
"Host": "1.1.1.1:11111",
"X-Forwarded-Host": "2.2.2.2,3.3.3.3,4.4.4.4"
}
consume(self.top.all.handleRequest(Client=("9.9.9.9", 9999), port=11111, Headers=Headers))
self.assertEqual(2, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[1].kwargs
self.assertEqual('4.4.4.4', handleRequestCallKwargs['Headers']['Host'])
self.assertEqual(80, handleRequestCallKwargs['port'])
def testHostMultiple(self):
self.createTree(deproxyForIpRanges=[
('9.9.9.0', '9.9.9.255')])
Headers={
"Host": ["1.1.1.1:11111", "4.4.4.4:44444"],
}
consume(self.top.all.handleRequest(Client=("9.9.9.9", 9999), port=11111, Headers=Headers))
self.assertEqual(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual("4.4.4.4:44444", handleRequestCallKwargs['Headers']['Host'])
def testDeproxyForIps(self):
self.createTree(deproxyForIps=['3.3.3.3'])
consume(self.top.all.handleRequest(
Client=("1.1.1.1", 11111),
Headers={"X-Forwarded-For": "2.2.2.2"}))
self.assertEqual(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual("1.1.1.1", handleRequestCallKwargs['Client'][0])
self.assertEqual({"X-Forwarded-For": "2.2.2.2"}, handleRequestCallKwargs['Headers'])
def testDeproxyUpdateIps(self):
# Expose updateIps from IpFilter
self.createTree(deproxyForIps=['127.7.7.7'])
# White box
allowDeproxying = lambda ip: self.deproxy._ipfilter.filterIpAddress(ip)
self.assertEqual(True, allowDeproxying('127.7.7.7'))
self.assertEqual(False, allowDeproxying('127.0.0.1'))
self.assertEqual(False, allowDeproxying('10.0.0.1'))
self.deproxy.updateIps(ipAddresses=['192.168.96.96'], ipRanges=[('10.0.0.0', '10.0.0.2')])
self.assertEqual(True, allowDeproxying('192.168.96.96'))
self.assertEqual(True, allowDeproxying('10.0.0.1'))
self.assertEqual(False, allowDeproxying('127.7.7.7'))
self.assertEqual(False, allowDeproxying('127.0.0.1'))
# Black box
consume(self.top.all.handleRequest(
Client=("192.168.96.96", 12345),
Headers={
"X-Forwarded-For": "2.2.2.2",
"X-Forwarded-Host": "example.org"}))
self.assertEqual(1, len(self.observer.calledMethods))
handleRequestCallKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual("2.2.2.2", handleRequestCallKwargs['Client'][0])
self.assertEqual('192.168.96.96', handleRequestCallKwargs['OriginalClient'][0])
self.assertEqual({"X-Forwarded-For": "2.2.2.2", "X-Forwarded-Host": "example.org", "Host": "example.org"}, handleRequestCallKwargs['Headers'])
def testOnlyDeproxied(self):
odp = OnlyDeproxied()
odp.addObserver(self.observer)
consume(odp.handleRequest(path='/path'))
self.assertEqual([], self.observer.calledMethodNames())
consume(odp.handleRequest(path='/path', OriginalClient=('1.2.3.4', 1234)))
self.assertEqual(['handleRequest'], self.observer.calledMethodNames())
self.assertEqual(dict(path='/path', OriginalClient=('1.2.3.4', 1234)), self.observer.calledMethods[0].kwargs)
