def createRequest(self, operation, request, outgoingData=None):
try:
# originatingObject = request.originatingObject()
# if originatingObject:
# varId = originatingObject.property('RAFT_requestId')
# if varId.isValid():
# print('createRequest', '%s->%s' % (str(varId.toString()), str(request.url().toEncoded().data().decode('utf-8'))))
# request.setAttribute(QtNetwork.QNetworkRequest.User + 1, varId)
# requestId = str(varId.toString())
# if requestId not in self.originalRequestIds:
# self.originalRequestIds.append(requestId)
# request.setAttribute(QtNetwork.QNetworkRequest.User + 2, varId)
url = request.url().toEncoded().data().decode('utf-8')
if outgoingData is not None and type(outgoingData) == QIODevice:
outgoingData = InterceptFormData(outgoingData)
return StoreNetworkReply(
self.framework, url, operation, request, outgoingData,
self.cookieJar(),
QNetworkAccessManager.createRequest(self, operation, request,
outgoingData), self)
except Exception as error:
# exceptions will cause a segfault
import traceback
print(('--->FIX ME:\n%s' % traceback.format_exc(error)))
request.setUrl(QUrl('about:blank'))
return QNetworkAccessManager.createRequest(self, operation,
request, outgoingData)
def createRequest(self, operation, request, outgoingData = None):
try:
# originatingObject = request.originatingObject()
# if originatingObject:
# varId = originatingObject.property('RAFT_requestId')
# if varId.isValid():
# print('createRequest', '%s->%s' % (str(varId.toString()), str(request.url().toEncoded().data().decode('utf-8'))))
# request.setAttribute(QtNetwork.QNetworkRequest.User + 1, varId)
# requestId = str(varId.toString())
# if requestId not in self.originalRequestIds:
# self.originalRequestIds.append(requestId)
# request.setAttribute(QtNetwork.QNetworkRequest.User + 2, varId)
url = request.url().toEncoded().data().decode('utf-8')
if outgoingData is not None and type(outgoingData) == QIODevice:
outgoingData = InterceptFormData(outgoingData)
return StoreNetworkReply(self.framework, url, operation, request, outgoingData, self.cookieJar(),
QNetworkAccessManager.createRequest(self, operation, request, outgoingData), self)
except Exception as error:
# exceptions will cause a segfault
import traceback
print(('--->FIX ME:\n%s' % traceback.format_exc(error)))
request.setUrl(QUrl('about:blank'))
return QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
def createRequest(self, operation, request, data):
url = request.url()
#print url
if operation == self.GetOperation:
actionName = str(url.scheme())
methodName = str(url.encodedHost())
is_action = False
if url.scheme() == 'file':
is_action = str(url.toEncoded()).endswith(".action")
if is_action is False:
return QNetworkAccessManager.createRequest( self, operation, request )
if is_action:
fileAction = str(url.toEncoded()).rpartition("/")[2].split(".")
actionName = fileAction[0]
methodName = fileAction[1]
result = self.__invokeAction(actionName, methodName, url)
if result is not None:
return HtmlReply(url, operation, result)
return QNetworkAccessManager.createRequest( self, operation, request )
def createRequest(self, op, request, device=None):
global lst, pg
try:
#print(str(request.url()))
urlLnk = (request.url()).toString()
#print(urlLnk)
path = str(request.url().path())
except UnicodeEncodeError:
pass
#print(path)
lower_case = path.lower()
#lst = tuple(open("easylist.txt", 'r'))
lst = [
"doubleclick.net", "ads", '.jpg', '.png', '.gif', '.css',
'facebook', '.aspx', r"||youtube-nocookie.com/gen_204?",
r"youtube.com###watch-branded-actions", "imagemapurl",
"b.scorecardresearch.com", "rightstuff.com", "scarywater.net",
"popup.js", "banner.htm", "_tribalfusion",
"||n4403ad.doubleclick.net^$third-party", ".googlesyndication.com",
"graphics.js", "fonts.googleapis.com/css", "s0.2mdn.net",
"server.cpmstar.com", "||banzai/banner.$subdocument",
"@@||anime-source.com^$document", "/pagead2.", "frugal.gif",
"jriver_banner.png", "show_ads.js",
'##a[href^="http://billing.frugalusenet.com/"]',
"http://jriver.com/video.html", "||animenewsnetwork.com^*.aframe?",
"||contextweb.com^$third-party", ".gutter", ".iab",
'http://www.animenewsnetwork.com/assets/[^"]*.jpg'
]
block = False
for l in lst:
if lower_case.find(l) != -1:
block = True
break
if block:
#print ("Skipping")
#print (request.url().path())
return QNetworkAccessManager.createRequest(
self, QNetworkAccessManager.GetOperation,
QtNetwork.QNetworkRequest(QtCore.QUrl()))
else:
if 'itag=' in urlLnk and 'redirector' not in urlLnk:
print('*********')
#print()
#print(path)
#print(urlLnk)
#pg.web.setHtml('<html>got the link</html>')
#self.lnkUrl.emit(path)
f = open('/tmp/AnimeWatch/lnk.txt', 'w')
f.write(urlLnk)
f.close()
return QNetworkAccessManager.createRequest(
self, op, request, device)
#return QNetworkAccessManager.createRequest(self, op, request, device)
#return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl()))
else:
return QNetworkAccessManager.createRequest(
self, op, request, device)
def createRequest(self, op, request, device=None):
url = request.url().toString()
doFilter = adblockFilter.match(url)
if doFilter:
try:
return QNetworkAccessManager.createRequest(self, self.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl()))
except:
print("failed to create request ...")
return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(nself, op, req, *args, **kwargs):
url = req.url().toString()
self._logger.debug("Google Translate: %s", url)
if '/translate_tts?' not in url:
return QNetworkAccessManager.createRequest(
nself, op, req, *args, **kwargs)
if self._cb and 'consumed' not in self._cb and \
'ttsspeed=' not in url: # avoid the slower audio playback
self._cb['consumed'] = True
rep = QNetworkAccessManager.createRequest(
nself, op, req, *args, **kwargs)
def finished():
try:
if not self._cb:
pass
elif rep.error():
raise RuntimeError("error in network reply")
elif rep.header(HEADER_LOCATION):
raise RuntimeError("got redirected away")
elif rep.header(
HEADER_CONTENT_TYPE) != 'audio/mpeg':
raise RuntimeError("unexpected Content-Type")
elif rep.header(HEADER_CONTENT_LENGTH) < 1024:
raise RuntimeError(
"Content-Length is too small")
else:
stream = rep.readAll()
if not stream:
raise RuntimeError("no stream returned")
elif len(stream) < 1024:
raise RuntimeError("stream is too small")
else:
self._cb['okay'](stream)
except StandardError as error:
try:
self._cb['fail'](format(error))
except StandardError:
pass
finally:
try:
rep.close()
except StandardError:
pass
rep.finished.connect(finished)
return QNetworkAccessManager.createRequest(
nself, op, FAKE, *args, **kwargs)
def createRequest(self, operation, request, device):
if request.url().scheme() != "gopher":
return QNetworkAccessManager.createRequest(self, operation,
request, device)
if operation == self.GetOperation:
# Handle gopher:// URLs separately by creating custom QNetworkReply
# objects.
reply = GopherReply(request.url())
return reply
else:
return QNetworkAccessManager.createRequest(self, operation,
request, device)
def createRequest(self, op, request, device = None ):
path = str(request.url().path())
lower_case = path.lower()
lst = ["banner", "ads", r"||youtube-nocookie.com/gen_204?", r"youtube.com###watch-branded-actions", "imagemapurl"]
block = False
for l in lst:
if lower_case.find(l) != -1:
block = True
break
if block:
print "Skipping "+request.url().path()
return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl()))
else:
return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(self, operation, request, outgoingData = None):
reply = None
try:
requestUrl = request.url()
scheme = str(requestUrl.scheme())
if scheme in ('data', 'about', 'javascript'):
reply = QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
elif scheme in ('http', 'https'):
url = requestUrl.toEncoded().data().decode('utf-8')
url_response = None
if url in self.request_lookaside:
url_response = self.request_lookaside[url]
else:
responses = []
for row in self.Data.read_responses_by_url(self.cursor, url):
responseItems = interface.data_row_to_response_items(row)
response_length = str(responseItems[ResponsesTable.RES_LENGTH])
if response_length and int(response_length) > 0 and str(responseItems[ResponsesTable.STATUS]).startswith('2'):
responses.append(responseItems)
if len(responses) > 0:
url_response = responses[-1]
# TODO: implement max size limit
self.request_lookaside[url] = url_response
if not url_response:
# try again, with base url
if '?' in url:
base_url = url[0:url.index('?')]
else:
base_url = url
for row in self.Data.read_responses_starting_with_url(self.cursor, base_url):
responseItems = interface.data_row_to_response_items(row)
response_length = str(responseItems[ResponsesTable.RES_LENGTH])
if response_length and int(response_length) > 0 and str(responseItems[ResponsesTable.STATUS]).startswith('2'):
responses.append(responseItems)
if len(responses) > 0:
url_response = responses[-1]
self.request_lookaside[url] = url_response
if url_response:
reply = CustomNetworkReply(self, requestUrl, url_response[ResponsesTable.RES_HEADERS], url_response[ResponsesTable.RES_DATA])
except Exception as error:
self.framework.report_implementation_error(error)
if not reply:
# must always return reply
request.setUrl(QUrl('about:blank'))
reply = QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
return reply
def createRequest(self, operation, request, outgoingData=None):
try:
url = request.url().toEncoded().data().decode('utf-8')
if outgoingData is not None and type(outgoingData) == QIODevice:
outgoingData = InterceptFormData(outgoingData)
return StoreNetworkReply(
self.framework, url, operation, request, outgoingData,
self.cookieJar(),
QNetworkAccessManager.createRequest(self, operation, request,
outgoingData), self)
except Exception as error:
# exceptions will cause a segfault
self.framework.report_exception(error)
request.setUrl(QUrl('about:blank'))
return QNetworkAccessManager.createRequest(self, operation,
request, outgoingData)
def createRequest(self, operation, request, data):
try:
requestFunc = self._dispatcher[request.url().scheme()]
except KeyError:
self.addHeadersToRequest(request)
return QNetworkAccessManager.createRequest(self, operation, request, data)
return requestFunc(self, operation, request, data)
def createRequest(self, operation, request, data):
method = self.get_method_name(operation)
allowed = True
req_url = request.url().toString()
if req_url.startswith('data:'):
req_url = 'data:[%s...]' % req_url[:50]
if operation == self.GetOperation:
if not self.is_request_allowed(request):
request.setUrl(QUrl('forbidden://localhost/'))
allowed = False
if allowed:
if self.proxy().hostName():
suffix = ' via proxy %s:%d' % (self.proxy().hostName(),
self.proxy().port())
else:
suffix = ''
logger.debug('%s [%d] %s%s' % (method.upper(), next(REQ_COUNTER),
req_url, suffix))
else:
logger_rules.debug('REJECT REQ %s' % req_url)
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
#request.setRawHeader('Accept-Encoding', 'gzip')
reply = QNetworkAccessManager.createRequest(self, operation, request, data)
reply.error.connect(self.handle_network_reply_error)
reply.readyRead.connect(lambda x=reply, y=copy(self.traffic_rules):\
handle_reply_ready_read(x, y))
return reply
def createRequest(self, operation, request, data):
if request.url().scheme() == "txmt":
self.commandUrlRequested.emit(request.url())
elif request.url().scheme() == "tm-file" and operation == self.GetOperation:
reply = TmFileReply(self, request.url(), self.GetOperation)
return reply
return QNetworkAccessManager.createRequest(self, operation, request, data)
def createRequest(self, op, req, outgoingData):
do_action("NetworkAccessManagerCreateRequestPre", Bunch(locals()))
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
if self.m_ignoreSslErrors:
reply.ignoreSslErrors()
headers = []
for header in req.rawHeaderList():
header = {"name": str(header), "value": str(req.rawHeader(header))}
headers.append(header)
self.m_idCounter += 1
self.m_ids[reply] = self.m_idCounter
data = {
"id": self.m_idCounter,
"url": req.url().toString(),
"method": toString(op),
"headers": headers,
"time": QDateTime.currentDateTime(),
}
reply.readyRead.connect(self.handleStarted)
do_action("NetworkAccessManagerCreateRequestPost", Bunch(locals()))
self.resourceRequested.emit(data)
return reply
def createRequest(self, operation, request, data):
if operation == self.GetOperation:
if not self.is_request_allowed(request):
request.setUrl(QUrl('forbidden://localhost/'))
else:
logger.debug(u'Querying URL: %s' % request.url().toString())
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
reply = QNetworkAccessManager.createRequest(self, operation, request, data)
reply.error.connect(self.catch_error)
reply = KitNetworkReply(self, reply)
# add Base-Url header, then we can get it from QWebView
# WTF?
if isinstance(request.originatingObject(), QWebFrame):
try:
reply.setRawHeader(
QByteArray('Base-Url'),
QByteArray('').append(request.originatingObject().page()
.mainFrame().baseUrl().toString()))
except Exception as e:
logger.debug(e)
return reply
def createRequest(self, operation, request, data):
if operation == self.GetOperation:
if self.is_forbidden(request):
# deny GET request for banned media type by setting dummy URL
# XXX abort properly
request.setUrl(QUrl(QString('forbidden://localhost/')))
else:
common.logger.debug(
common.to_unicode(
request.url().toString().toUtf8().data()).encode(
'utf-8'))
#print request.url().toString(), operation
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
reply = QNetworkAccessManager.createRequest(self, operation, request,
data)
reply.error.connect(self.catch_error)
#add Base-Url header, then we can get it from QWebView
if isinstance(request.originatingObject(), QWebFrame):
try:
reply.setRawHeader(
QByteArray('Base-Url'),
QByteArray('').append(request.originatingObject().page().
mainFrame().baseUrl().toString()))
except Exception, e:
common.logger.debug(e)
def createRequest(self, op, req, outgoingData):
do_action('NetworkAccessManagerCreateRequestPre')
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
if self.m_ignoreSslErrors:
reply.ignoreSslErrors()
headers = []
for header in req.rawHeaderList():
header = {
'name': str(header),
'value': str(req.rawHeader(header))
}
headers.append(header)
self.m_idCounter += 1
self.m_ids[reply] = self.m_idCounter
data = {
'id': self.m_idCounter,
'url': req.url().toString(),
'method': self.operationToString(op),
'headers': headers,
'time': QDateTime.currentDateTime()
}
reply.readyRead.connect(self.handleStarted)
do_action('NetworkAccessManagerCreateRequestPost')
self.resourceRequested.emit(data)
return reply
def createRequest(self, operation, request, post):
"""Override creating a network request
"""
url = request.url().toString()
if str(request.url().path()).endswith('.ttf'):
# block fonts, which can cause webkit to crash
common.logger.debug(u'Blocking: {}'.format(url))
request.setUrl(QUrl())
data = post if post is None else post.peek(MAX_POST_SIZE)
key = u'{} {}'.format(url, data)
use_cache = not url.startswith('file')
if self.cache is not None and use_cache and key in self.cache:
common.logger.debug(u'Load from cache: {}'.format(key))
content, headers, attributes = self.cache[key]
reply = CachedNetworkReply(self, request.url(), content, headers,
attributes)
else:
common.logger.debug(u'Request: {} {}'.format(url, post or ''))
reply = QNetworkAccessManager.createRequest(
self, operation, request, post)
reply.error.connect(self.catch_error)
self.active_requests.append(reply)
reply.destroyed.connect(self.active_requests.remove)
# save reference to original request
reply.content = QByteArray()
reply.readyRead.connect(self._save_content(reply))
if self.cache is not None and use_cache:
reply.finished.connect(self._cache_content(reply, key))
reply.orig_request = request
reply.data = self.parse_data(data)
return reply
def createRequest(self, op, req, outgoingData):
do_action('NetworkAccessManagerCreateRequestPre')
reply = QNetworkAccessManager.createRequest(self, op, req,
outgoingData)
if self.m_ignoreSslErrors:
reply.ignoreSslErrors()
headers = []
for header in req.rawHeaderList():
header = {'name': str(header), 'value': str(req.rawHeader(header))}
headers.append(header)
self.m_idCounter += 1
self.m_ids[reply] = self.m_idCounter
data = {
'id': self.m_idCounter,
'url': req.url().toString(),
'method': toString(op),
'headers': headers,
'time': QDateTime.currentDateTime()
}
reply.readyRead.connect(self.handleStarted)
do_action('NetworkAccessManagerCreateRequestPost')
self.resourceRequested.emit(data)
return reply
def createRequest(self, operation, request, data):
try:
requestFunc = self._dispatcher[request.url().scheme()]
except KeyError:
self.addHeadersToRequest(request)
return QNetworkAccessManager.createRequest(self, operation, request, data)
return requestFunc(self, operation, request, data)
def createRequest(self, op, req, outgoingData):
if op == QNetworkAccessManager.GetOperation:
qDebug('HTTP/1.1 GET Request')
elif op == QNetworkAccessManager.PostOperation:
qDebug('HTTP/1.1 POST Request')
elif op == QNetworkAccessManager.HeadOperation:
qDebug('HTTP/1.1 HEAD Request')
elif op == QNetworkAccessManager.PutOperation:
qDebug('HTTP/1.1 PUT Request')
elif op == QNetworkAccessManager.DeleteOperation:
qDebug('HTTP/1.1 DELETE Request')
elif op == QNetworkAccessManager.CustomOperation:
qDebug('HTTP/1.1 CUSTOM Request')
else:
qWarning('Unexpected HTTP Operation Type')
qDebug('URL %s' % req.url().toString())
do_action('NetworkAccessManagerCreateRequestPre', Bunch(locals()))
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
if self.m_ignoreSslErrors == 'yes':
reply.ignoreSslErrors()
do_action('NetworkAccessManagerCreateRequestPost', Bunch(locals()))
return reply
def createRequest(self, operation, request, data):
if operation == self.GetOperation:
if not self.is_request_allowed(request):
request.setUrl(QUrl('forbidden://localhost/'))
else:
logger.debug(u'Quering URL: %s' % request.url().toString())
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
reply = QNetworkAccessManager.createRequest(self, operation, request,
data)
reply.error.connect(self.catch_error)
reply = KitNetworkReply(self, reply)
# add Base-Url header, then we can get it from QWebView
# WTF?
if isinstance(request.originatingObject(), QWebFrame):
try:
reply.setRawHeader(
QByteArray('Base-Url'),
QByteArray('').append(request.originatingObject().page().
mainFrame().baseUrl().toString()))
except Exception as e:
logger.debug(e)
return reply
def createRequest(self, operation, request, device):
path = o = request.url().toString()
if path.startswith('app://') or path.startswith('lens://'):
if path == 'app:///':
path = 'file://' + self._uri_app_base + 'app.html'
logger.debug('Loading app resource: {0} ({1})'.format(o, path))
elif path.startswith('app://'):
path = path.replace('app://', 'file://' + self._uri_app_base)
logger.debug('Loading app resource: {0} ({1})'.format(o, path))
# variable substitution
path = path.replace('$backend', 'qt4')
elif path.startswith('lens://'):
path = path.replace('lens://', 'file://' + self._uri_lens_base)
# make lens.css backend specific
path = path.replace('lens.css', 'lens-qt4.css')
logger.debug('Loading lens resource: {0} ({1})'.format(o, path))
request.setUrl(QUrl(QString(path)))
return QNetworkAccessManager.createRequest(self, operation, request, device)
def createRequest(self, operation, request, device):
path = o = request.url().toString()
if path.startswith('app://') or path.startswith('lens://'):
if path == 'app:///':
path = 'file://' + self._uri_app_base + 'app.html'
logger.debug('Loading app resource: {0} ({1})'.format(o, path))
elif path.startswith('app://'):
path = path.replace('app://', 'file://' + self._uri_app_base)
logger.debug('Loading app resource: {0} ({1})'.format(o, path))
# variable substitution
path = path.replace('$backend', 'qt4')
elif path.startswith('lens://'):
path = path.replace('lens://', 'file://' + self._uri_lens_base)
# make lens.css backend specific
path = path.replace('lens.css', 'lens-qt4.css')
logger.debug('Loading lens resource: {0} ({1})'.format(
o, path))
request.setUrl(QUrl(QString(path)))
return QNetworkAccessManager.createRequest(self, operation, request,
device)
def createRequest(self, operation, request, data):
#print("mymanager handles ", request.url())
if self.referer and self.webpage.history().canGoBack():
request.setRawHeader('Referer', self.referer)
for key in self.header.keys():
request.setRawHeader(key, self.header[key])
return QNetworkAccessManager.createRequest(self, operation, request,
data)
def createRequest(self, op, request, outgoingData=None):
"""
Protected method to create a request.
@param op the operation to be performed (QNetworkAccessManager.Operation)
@param request reference to the request object (QNetworkRequest)
@param outgoingData reference to an IODevice containing data to be sent
(QIODevice)
@return reference to the created reply object (QNetworkReply)
"""
scheme = request.url().scheme()
if scheme == "https" and (not SSL_AVAILABLE or not QSslSocket.supportsSsl()):
return NetworkProtocolUnknownErrorReply(scheme, self)
if op == QNetworkAccessManager.PostOperation and outgoingData is not None:
outgoingDataByteArray = outgoingData.peek(1024 * 1024)
Helpviewer.HelpWindow.HelpWindow.passwordManager().post(request, outgoingDataByteArray)
reply = None
if unicode(scheme) in self.__schemeHandlers:
reply = self.__schemeHandlers[unicode(scheme)].createRequest(op, request, outgoingData)
if reply is not None:
return reply
if not self.__acceptLanguage.isEmpty():
req = QNetworkRequest(request)
req.setRawHeader("Accept-Language", self.__acceptLanguage)
else:
req = request
# AdBlock code
if op == QNetworkAccessManager.GetOperation:
if self.__adblockNetwork is None:
self.__adblockNetwork = Helpviewer.HelpWindow.HelpWindow.adblockManager().network()
reply = self.__adblockNetwork.block(req)
if reply is not None:
reply.setParent(self)
return reply
# set cache policy
if op == QNetworkAccessManager.GetOperation:
urlHost = req.url().host()
for host in self.NoCacheHosts:
if host in urlHost:
req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork)
break
else:
req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork)
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
self.emit(
SIGNAL("requestCreated(QNetworkAccessManager::Operation, const QNetworkRequest&, QNetworkReply*)"),
op,
req,
reply,
)
return reply
def createRequest(self, operation, request, data):
if operation == self.GetOperation:
if request.url().scheme() == self.url_scheme:
# handle epwing:// urls separately by creating custom QNetworkReply objects.
reply = EpwingReply(self, request, request.url(), self.GetOperation, self.book_manager)
return reply
#return self.old_manager.createRequest(operation, request, data)
return QNetworkAccessManager.createRequest(self, operation, request, data)
def createRequest(self, op, request, device = None ):
global lst
try:
path = str(request.url().path())
except UnicodeEncodeError:
pass
lower_case = path.lower()
#lst = tuple(open("easylist.txt", 'r'))
lst = ["doubleclick.net" ,"ads",'.jpg','redirector','itag=','.png','.gif','.css','google','facebook','.aspx',r"http[^'].mp4",r"http[^'].flv", r"||youtube-nocookie.com/gen_204?", r"youtube.com###watch-branded-actions", "imagemapurl","b.scorecardresearch.com","rightstuff.com","scarywater.net","popup.js","banner.htm","_tribalfusion","||n4403ad.doubleclick.net^$third-party",".googlesyndication.com","graphics.js","fonts.googleapis.com/css","s0.2mdn.net","server.cpmstar.com","||banzai/banner.$subdocument","@@||anime-source.com^$document","google","/pagead2.","frugal.gif","jriver_banner.png","show_ads.js",'##a[href^="http://billing.frugalusenet.com/"]',"http://jriver.com/video.html","||animenewsnetwork.com^*.aframe?","||contextweb.com^$third-party",".gutter",".iab",'http://www.animenewsnetwork.com/assets/[^"]*.jpg']
block = False
for l in lst:
if lower_case.find(l) != -1:
block = True
break
if block:
print ("Skipping")
print (request.url().path())
return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl()))
else:
return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(self, operation, request, outgoingData=None):
try:
url = request.url().toEncoded().data().decode("utf-8")
if outgoingData is not None and type(outgoingData) == QIODevice:
outgoingData = InterceptFormData(outgoingData)
return StoreNetworkReply(
self.framework,
url,
operation,
request,
outgoingData,
self.cookieJar(),
QNetworkAccessManager.createRequest(self, operation, request, outgoingData),
self,
)
except Exception as error:
# exceptions will cause a segfault
self.framework.report_exception(error)
request.setUrl(QUrl("about:blank"))
return QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
def createRequest(self, op, request, device):
# prefer valid cache instead of network
self.reqCount = self.reqCount + 1
if re.match(".*?pubads\.g\.doubleclick\.net*", request.url().toString()):
request.setUrl(QUrl("http://img6a.flixcart.com/www/prod/images/flipkart_india-e5f5aa9f.png"))
# print "REQ :",self.reqCount," ",request.url()
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache)
reply = QNetworkAccessManager.createRequest(self, op, request, device)
reply.error.connect(self.logNetworkErrors)
return reply
def createRequest(self, op, request, device=None):
global block_list
try:
path = str(request.url().toString())
except UnicodeEncodeError:
path = str(request.url().path())
pass
lower_path = path.lower()
#lst = tuple(open("easylist.txt", 'r'))
block_list = [
"doubleclick.net", "ads", r"||youtube-nocookie.com/gen_204?",
r"youtube.com###watch-branded-actions", "imagemapurl",
"b.scorecardresearch.com", "rightstuff.com", "scarywater.net",
"popup.js", "banner.htm", "_tribalfusion",
"||n4403ad.doubleclick.net^$third-party", ".googlesyndication.com",
"graphics.js", "fonts.googleapis.com/css", "s0.2mdn.net",
"server.cpmstar.com", "||banzai/banner.$subdocument",
"@@||anime-source.com^$document", "/pagead2.", "frugal.gif",
"jriver_banner.png", "show_ads.js",
'##a[href^="http://billing.frugalusenet.com/"]',
"http://jriver.com/video.html", "||animenewsnetwork.com^*.aframe?",
"||contextweb.com^$third-party", ".gutter", ".iab",
'http://www.animenewsnetwork.com/assets/[^"]*.jpg', 'revcontent'
]
block = False
for l in block_list:
if l in lower_path:
block = True
break
if block:
print("Skipping")
print(path)
return QNetworkAccessManager.createRequest(
self, QNetworkAccessManager.GetOperation,
QtNetwork.QNetworkRequest(QtCore.QUrl()))
else:
return QNetworkAccessManager.createRequest(self, op, request,
device)
def _manager_create_request(self, operation, request, data):
url = unicode(request.url().toString())
operation_name = self._operation_names[operation].upper()
self._debug(INFO, "Request: %s %s" % (operation_name, url))
for h in request.rawHeaderList():
self._debug(DEBUG, " %s: %s" % (h, request.rawHeader(h)))
if self._url_filter:
if self._url_filter(self._operation_names[operation], url) is False:
self._debug(INFO, "URL filtered: %s" % url)
request.setUrl(QUrl("about:blank"))
else:
self._debug(DEBUG, "URL not filtered: %s" % url)
reply = QNetworkAccessManager.createRequest(self.manager, operation, request, data)
return reply
def createRequest(self, op, request, device = None): if InjectedQNetworkRequest.thatRequestHasMagicQt4(request): r = InjectedNetworkReply(self, request.url(), self.response.read(), op) default_cookie_domain = self._cookie_default_domain(request) cookiejar = self._createCookieJarfromInjectedResponse(default_cookie_domain) self.setCookieJar(cookiejar) else: r = QNetworkAccessManager.createRequest(self, op, request, device) r.finished.connect(self.checkAutoCloseUrls) return r
def _manager_create_request(self, operation, request, data):
url = unicode(request.url().toString())
operation_name = self._operation_names[operation].upper()
self._debug(INFO, "Request: %s %s" % (operation_name, url))
for h in request.rawHeaderList():
self._debug(DEBUG, " %s: %s" % (h, request.rawHeader(h)))
if self._url_filter:
if self._url_filter(self._operation_names[operation],
url) is False:
self._debug(INFO, "URL filtered: %s" % url)
request.setUrl(QUrl("about:blank"))
else:
self._debug(DEBUG, "URL not filtered: %s" % url)
reply = QNetworkAccessManager.createRequest(self.manager, operation,
request, data)
return reply
def createRequest(self, op, request, device = None): if InjectedQNetworkRequest.thatRequestHasMagicQt4(request): r = InjectedNetworkReply(self, request.url(), self.urllib_response.read(), op, self.urllib_request, self.urllib_response) default_cookie_domain = self._cookie_default_domain(request) cookiejar = self._import_cookie_jar(self.http_cookie_jar, default_cookie_domain) self.setCookieJar(cookiejar) else: self.urllib_response = None self.urllib_request = None original_r = QNetworkAccessManager.createRequest(self, op, request, device) original_r.sslErrors.connect(self.sslErrorHandler) r = SniffingNetworkReply(self, request, original_r, op) r.finished.connect(self.requestFinishedActions) return r
def createRequest(self, op, request, outgoingData):
if op == QNetworkAccessManager.GetOperation:
try:
urlPath = unicode(request.url().path()).encode(
'UTF-8', 'replace')
if urlPath.rfind('.js') != -1 and urlPath.rfind(
'assets') == -1:
reply = ReportJSRenderReply(self, request)
return reply
elif request.url().scheme() == "dff-node-thumbnail":
reply = NodeThumbnailRenderReply(self, request)
return reply
except Exception as e:
print 'UrlRenderer error: ' + str(e)
request = QNetworkAccessManager.createRequest(self, op, request,
outgoingData)
return request
def createRequest(self, op, request, outgoingData = None):
"""
Protected method to create a request.
@param op the operation to be performed (QNetworkAccessManager.Operation)
@param request reference to the request object (QNetworkRequest)
@param outgoingData reference to an IODevice containing data to be sent
(QIODevice)
@return reference to the created reply object (QNetworkReply)
"""
if self.primaryManager is not None and \
self.__webPage is not None:
pageRequest = QNetworkRequest(request)
self.__webPage.populateNetworkRequest(pageRequest)
return self.primaryManager.createRequest(op, pageRequest, outgoingData)
else:
return QNetworkAccessManager.createRequest(self, op, request, outgoingData)
def createRequest(self, op, request, device):
# prefer valid cache instead of network
self.reqCount = self.reqCount + 1
if re.match(".*?pubads\.g\.doubleclick\.net*",
request.url().toString()):
request.setUrl(
QUrl(
"http://img6a.flixcart.com/www/prod/images/flipkart_india-e5f5aa9f.png"
))
# print "REQ :",self.reqCount," ",request.url()
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
reply = QNetworkAccessManager.createRequest(self, op, request, device)
reply.error.connect(self.logNetworkErrors)
return reply
def createRequest(self, op, req, outgoingData):
if op == QNetworkAccessManager.GetOperation:
qDebug('HTTP/1.1 GET Request')
elif op == QNetworkAccessManager.PostOperation:
qDebug('HTTP/1.1 POST Request')
elif op == QNetworkAccessManager.HeadOperation:
qDebug('HTTP/1.1 HEAD Request')
elif op == QNetworkAccessManager.PutOperation:
qDebug('HTTP/1.1 PUT Request')
elif op == QNetworkAccessManager.DeleteOperation:
qDebug('HTTP/1.1 DELETE Request')
elif op == QNetworkAccessManager.CustomOperation:
qDebug('HTTP/1.1 CUSTOM Request')
else:
qWarning('Unexpected HTTP Operation Type')
qDebug('URL %s' % req.url().toString())
return QNetworkAccessManager.createRequest(self, op, req, outgoingData)
def createRequest(self, op, req, outgoingData):
if op == QNetworkAccessManager.GetOperation:
qDebug('HTTP/1.1 GET Request')
elif op == QNetworkAccessManager.PostOperation:
qDebug('HTTP/1.1 POST Request')
elif op == QNetworkAccessManager.HeadOperation:
qDebug('HTTP/1.1 HEAD Request')
elif op == QNetworkAccessManager.PutOperation:
qDebug('HTTP/1.1 PUT Request')
elif op == QNetworkAccessManager.DeleteOperation:
qDebug('HTTP/1.1 DELETE Request')
elif op == QNetworkAccessManager.CustomOperation:
qDebug('HTTP/1.1 CUSTOM Request')
else:
qWarning('Unexpected HTTP Operation Type')
qDebug('URL %s' % req.url().toString())
return QNetworkAccessManager.createRequest(self, op, req, outgoingData)
def createRequest(self, operation, request, data):
if operation == self.GetOperation:
if self.is_forbidden(request):
# deny GET request for banned media type by setting dummy URL
# XXX abort properly
request.setUrl(QUrl('forbidden://localhost/'))
else:
common.logger.debug(common.to_unicode(request.url().toString()).encode('utf-8'))
#print request.url().toString(), operation
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache)
reply = QNetworkAccessManager.createRequest(self, operation, request, data)
reply.error.connect(self.catch_error)
# add Base-Url header, then we can get it from QWebView
if isinstance(request.originatingObject(), QWebFrame):
try:
reply.setRawHeader(QByteArray('Base-Url'), QByteArray('').append(request.originatingObject().page().mainFrame().baseUrl().toString()))
except Exception, e:
common.logger.debug(e)
def createRequest(self, operation, request, data):
if operation == self.GetOperation:
if self.is_forbidden(request):
# deny GET request for banned media type by setting dummy URL
request.setUrl(QUrl(QString('forbidden://localhost/')))
else:
print request.url().toString()
else:
pass
#print 'POST'
#print request.url().toString()
#data.open(QIODevice.ReadOnly)
#print data.readAll()
#print data.peek(100000000000)
#data.seek(0)
#data.close()
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache)
reply = QNetworkAccessManager.createRequest(self, operation, request, data)
#reply.finished.connect(self.catch_finished)
reply.error.connect(self.catch_error)
return reply
def createRequest(self, operation, request, data):
#request.setRawHeader("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
#print 'createRequest: url is '+request.url()
#print 'mymanager handles ',request.url()
#file = open('request.html', 'a')
#file.write(request.url().toString()+'\n======================================================================\n\n\n\n\n\n\n')
#file.close()
request.setRawHeader("Connection","keep-alive");
request.setRawHeader("Host", "google.com")
request.setRawHeader("Proxy-Connection", "keep-alive")
request.setRawHeader("Cache-Control", "max-age=0")
request.setRawHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
request.setRawHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36")
request.setRawHeader("DNT", "1")
request.setRawHeader("Referer", "http://www.google.com")
request.setRawHeader("Accept-Encoding", "deflate, sdch")
request.setRawHeader("Accept-Language", "en-US,en;q=0.8")
request.setRawHeader("Cookie", "")
#a = request.rawHeader("Cookie")
#print a
sys.stdout.write('.') # progress bar
return QNetworkAccessManager.createRequest( self, operation, request, data )
def create_request(self, operation, request, data):
""" Reimplemented to intercept requests. Stops blacklisted requests,
matches requests with replies. Stores on a PostgreSQL database.
"""
if es_url_local(request.url()) and not es_font(request.url()):
return QNetworkAccessManager.createRequest(
self, operation, request, data)
if (usando_whitelist(self.whitelist, request.url()) or
es_font(request.url())):
print "FILTERING %s" % request.url().toString()[:255]
return QNetworkAccessManager.createRequest(
self,
operation,
QNetworkRequest(QUrl("about:blank")),
data)
request.setAttribute(
QNetworkRequest.HttpPipeliningAllowedAttribute, True)
response = QNetworkAccessManager.createRequest(
self, operation, request, data)
if operation == QNetworkAccessManager.PostOperation:
post_str = unicode(data.peek(4096))
data_json = filtra(parse_qsl(post_str, keep_blank_values=True))
else:
data_json = None
def indice(reply, idx):
""" This function returns a closure enclosing the current index
and its associated reply. This is required since there's no
knowing when (if at all) a request will be replied to; once it
happens, the index will surely have changed.
The 'cheatgc' list is required because of a bug where the reference
for the 'reply' and 'idx' objects is lost sometimes after 'ret'
completes, despite having created a closure referencing them. By
appending to the list, garbage collection is prevented.
"""
# please don't do garbage collection...
self.cheatgc.append(reply)
self.cheatgc.append(idx)
def ret():
""" Closure; freezes 'reply' and 'idx' so they will be
accessible at the time the request finalizes and this closure
is called.
"""
try:
encabezados = filtra(reply.rawHeaderPairs())
(statuscode, _) = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute).toInt()
if self.log:
self.log.store_reply({
"id": self.instance_id,
"idx": idx,
"scheme": unicode(reply.url().scheme()),
"host": unicode(reply.url().host()),
"path": unicode(reply.url().path()),
"query": filtra(reply.url().encodedQueryItems()),
"fragment": notnull(
unicode(reply.url().fragment())),
"status": statuscode,
"headers": encabezados
})
# ...until we're done with the request
# (pyqt/sip related trouble)
self.cheatgc.remove(reply)
self.cheatgc.remove(idx)
#print "[%s]" % (len(self.cheatgc))
except NameError as name_error:
print name_error
print "Except NameError!"
return ret
response.finished.connect(indice(response, self.count))
if self.log:
headers = []
for header in request.rawHeaderList():
headers.append((header, request.rawHeader(header)))
self.log.store_request({
"id": self.instance_id,
"idx": self.count,
"op": OPERATIONS[operation],
"scheme": unicode(request.url().scheme()),
"host": unicode(request.url().host()),
"path": unicode(request.url().path()),
"query": filtra(request.url().encodedQueryItems()),
"fragment": notnull(
unicode(request.url().fragment())),
"data": data_json,
"source": unicode(
request.originatingObject().requestedUrl().host()),
"headers": filtra(headers)
})
self.count += 1
return response
def createRequest(self, op, req, outgoingData):
for k in self.args:
req.setRawHeader(k, self.args[k] )
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
return reply
def createRequest(self, op, request, device=None):
request.setRawHeader(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'
)
return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(self, operation, request, data):
if request.url().scheme() != "tentia":
return QNetworkAccessManager.createRequest(self, operation, request, data)
else:
self.tentia_callback(request.url())
return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QNetworkRequest(QtCore.QUrl()))
def create_request(self, operation, request, data):
""" Reimplemented to intercept requests. Stops blacklisted requests,
matches requests with replies. Stores on a PostgreSQL database.
"""
if es_url_local(request.url()) and not es_font(request.url()):
return QNetworkAccessManager.createRequest(self, operation,
request, data)
if (usando_whitelist(self.whitelist, request.url())
or es_font(request.url())):
print "FILTERING %s" % request.url().toString()[:255]
return QNetworkAccessManager.createRequest(
self, operation, QNetworkRequest(QUrl("about:blank")), data)
request.setAttribute(QNetworkRequest.HttpPipeliningAllowedAttribute,
True)
response = QNetworkAccessManager.createRequest(self, operation,
request, data)
if operation == QNetworkAccessManager.PostOperation:
post_str = unicode(data.peek(4096))
data_json = filtra(parse_qsl(post_str, keep_blank_values=True))
else:
data_json = None
def indice(reply, idx):
""" This function returns a closure enclosing the current index
and its associated reply. This is required since there's no
knowing when (if at all) a request will be replied to; once it
happens, the index will surely have changed.
The 'cheatgc' list is required because of a bug where the reference
for the 'reply' and 'idx' objects is lost sometimes after 'ret'
completes, despite having created a closure referencing them. By
appending to the list, garbage collection is prevented.
"""
# please don't do garbage collection...
self.cheatgc.append(reply)
self.cheatgc.append(idx)
def ret():
""" Closure; freezes 'reply' and 'idx' so they will be
accessible at the time the request finalizes and this closure
is called.
"""
try:
encabezados = filtra(reply.rawHeaderPairs())
(statuscode, _) = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute).toInt()
if self.log:
self.log.store_reply({
"id":
self.instance_id,
"idx":
idx,
"scheme":
unicode(reply.url().scheme()),
"host":
unicode(reply.url().host()),
"path":
unicode(reply.url().path()),
"query":
filtra(reply.url().encodedQueryItems()),
"fragment":
notnull(unicode(reply.url().fragment())),
"status":
statuscode,
"headers":
encabezados
})
# ...until we're done with the request
# (pyqt/sip related trouble)
self.cheatgc.remove(reply)
self.cheatgc.remove(idx)
#print "[%s]" % (len(self.cheatgc))
except NameError as name_error:
print name_error
print "Except NameError!"
return ret
response.finished.connect(indice(response, self.count))
if self.log:
headers = []
for header in request.rawHeaderList():
headers.append((header, request.rawHeader(header)))
self.log.store_request({
"id":
self.instance_id,
"idx":
self.count,
"op":
OPERATIONS[operation],
"scheme":
unicode(request.url().scheme()),
"host":
unicode(request.url().host()),
"path":
unicode(request.url().path()),
"query":
filtra(request.url().encodedQueryItems()),
"fragment":
notnull(unicode(request.url().fragment())),
"data":
data_json,
"source":
unicode(request.originatingObject().requestedUrl().host()),
"headers":
filtra(headers)
})
self.count += 1
return response
def createRequest(self, operation, request, outgoingData=None):
reply = None
try:
requestUrl = request.url()
scheme = str(requestUrl.scheme())
if scheme in ('data', 'about', 'javascript'):
reply = QNetworkAccessManager.createRequest(
self, operation, request, outgoingData)
elif scheme in ('http', 'https'):
url = requestUrl.toEncoded().data().decode('utf-8')
url_response = None
if url in self.request_lookaside:
url_response = self.request_lookaside[url]
else:
responses = []
for row in self.Data.read_responses_by_url(
self.cursor, url):
responseItems = interface.data_row_to_response_items(
row)
response_length = str(
responseItems[ResponsesTable.RES_LENGTH])
if response_length and int(
response_length) > 0 and str(responseItems[
ResponsesTable.STATUS]).startswith('2'):
responses.append(responseItems)
if len(responses) > 0:
url_response = responses[-1]
# TODO: implement max size limit
self.request_lookaside[url] = url_response
if not url_response:
# try again, with base url
if '?' in url:
base_url = url[0:url.index('?')]
else:
base_url = url
for row in self.Data.read_responses_starting_with_url(
self.cursor, base_url):
responseItems = interface.data_row_to_response_items(
row)
response_length = str(
responseItems[ResponsesTable.RES_LENGTH])
if response_length and int(
response_length) > 0 and str(
responseItems[ResponsesTable.
STATUS]).startswith('2'):
responses.append(responseItems)
if len(responses) > 0:
url_response = responses[-1]
self.request_lookaside[url] = url_response
if url_response:
reply = CustomNetworkReply(
self, requestUrl,
url_response[ResponsesTable.RES_HEADERS],
url_response[ResponsesTable.RES_DATA])
except Exception as error:
self.framework.report_implementation_error(error)
if not reply:
# must always return reply
request.setUrl(QUrl('about:blank'))
reply = QNetworkAccessManager.createRequest(
self, operation, request, outgoingData)
return reply
def createRequest(self,op,request,outgoingdata):
scheme=request.url().scheme()
# print unicode(request.url().path())
if scheme==QLatin1String('ms-its'):
return PyChmNetReply(request,request.url(),self.qwebview,self.qwebview)
return QNetworkAccessManager.createRequest(self,op,request,outgoingdata)
def createRequest(self, operation, request, data): if request.url().scheme() != "bungloo": return QNetworkAccessManager.createRequest(self, operation, request, data) else: self.bungloo_callback(request.url()) return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QNetworkRequest(QtCore.QUrl()))