def __init__(self,
filters_path=None,
verbosity=None,
allowed_schemes=None):
verbosity = defaults.VERBOSITY if verbosity is None else verbosity
self.verbosity = verbosity
self.request_middlewares = []
self.response_middlewares = []
self.adblock_rules = None
# Initialize request and response middlewares
allowed_schemes = (defaults.ALLOWED_SCHEMES if allowed_schemes is None
else allowed_schemes.split(','))
if allowed_schemes:
self.request_middlewares.append(
AllowedSchemesMiddleware(allowed_schemes, verbosity=verbosity))
if self.verbosity >= 2:
self.request_middlewares.append(RequestLoggingMiddleware())
self.request_middlewares.append(
AllowedDomainsMiddleware(verbosity=verbosity))
self.request_middlewares.append(ResourceTimeoutMiddleware())
self.request_middlewares.append(ResponseBodyTrackingMiddleware())
if filters_path is not None:
self.adblock_rules = AdblockRulesRegistry(filters_path,
verbosity=verbosity)
self.request_middlewares.append(
AdblockMiddleware(self.adblock_rules, verbosity=verbosity))
self.response_middlewares.append(ContentTypeMiddleware(self.verbosity))
class SplashQNetworkAccessManager(ProxiedQNetworkAccessManager):
"""
This QNetworkAccessManager provides:
* proxy support;
* request middleware support;
* additional logging.
"""
def __init__(self, filters_path, allowed_schemes, verbosity):
super(SplashQNetworkAccessManager, self).__init__(verbosity=verbosity)
self.request_middlewares = []
if self.verbosity >= 2:
self.request_middlewares.append(RequestLoggingMiddleware())
if allowed_schemes:
self.request_middlewares.append(
AllowedSchemesMiddleware(allowed_schemes, verbosity=verbosity)
)
self.request_middlewares.append(AllowedDomainsMiddleware(verbosity=verbosity))
if filters_path is not None:
self.adblock_rules = AdblockRulesRegistry(filters_path, verbosity=verbosity)
self.request_middlewares.append(
AdblockMiddleware(self.adblock_rules, verbosity=verbosity)
)
else:
self.adblock_rules = None
def createRequest(self, operation, request, outgoingData=None):
splash_request = self._getSplashRequest(request)
if splash_request:
for filter in self.request_middlewares:
request = filter.process(request, splash_request, operation, outgoingData)
return super(SplashQNetworkAccessManager, self).createRequest(operation, request, outgoingData)
def unknownFilters(self, filter_names):
names = [f for f in filter_names.split(',') if f]
if self.adblock_rules is None:
return names
return [name for name in names
if not (self.adblock_rules.filter_is_known(name) or name=='none')]
def __init__(self, filters_path, allowed_schemes, verbosity):
super(SplashQNetworkAccessManager, self).__init__(verbosity=verbosity)
self.request_middlewares = []
if self.verbosity >= 2:
self.request_middlewares.append(RequestLoggingMiddleware())
if allowed_schemes:
self.request_middlewares.append(
AllowedSchemesMiddleware(allowed_schemes, verbosity=verbosity))
self.request_middlewares.append(
AllowedDomainsMiddleware(verbosity=verbosity))
if filters_path is not None:
self.adblock_rules = AdblockRulesRegistry(filters_path,
verbosity=verbosity)
self.request_middlewares.append(
AdblockMiddleware(self.adblock_rules, verbosity=verbosity))
def __init__(self, filters_path, allowed_schemes, verbosity):
super(SplashQNetworkAccessManager, self).__init__(verbosity=verbosity)
self.request_middlewares = []
if self.verbosity >= 2:
self.request_middlewares.append(RequestLoggingMiddleware())
if allowed_schemes:
self.request_middlewares.append(
AllowedSchemesMiddleware(allowed_schemes, verbosity=verbosity)
)
self.request_middlewares.append(AllowedDomainsMiddleware(verbosity=verbosity))
if filters_path is not None:
self.adblock_rules = AdblockRulesRegistry(filters_path, verbosity=verbosity)
self.request_middlewares.append(
AdblockMiddleware(self.adblock_rules, verbosity=verbosity)
)
else:
self.adblock_rules = None