def createRequest(self, operation, request, outgoingData=None): try: # originatingObject = request.originatingObject() # if originatingObject: # varId = originatingObject.property('RAFT_requestId') # if varId.isValid(): # print('createRequest', '%s->%s' % (str(varId.toString()), str(request.url().toEncoded().data().decode('utf-8')))) # request.setAttribute(QtNetwork.QNetworkRequest.User + 1, varId) # requestId = str(varId.toString()) # if requestId not in self.originalRequestIds: # self.originalRequestIds.append(requestId) # request.setAttribute(QtNetwork.QNetworkRequest.User + 2, varId) url = request.url().toEncoded().data().decode('utf-8') if outgoingData is not None and type(outgoingData) == QIODevice: outgoingData = InterceptFormData(outgoingData) return StoreNetworkReply( self.framework, url, operation, request, outgoingData, self.cookieJar(), QNetworkAccessManager.createRequest(self, operation, request, outgoingData), self) except Exception as error: # exceptions will cause a segfault import traceback print(('--->FIX ME:\n%s' % traceback.format_exc(error))) request.setUrl(QUrl('about:blank')) return QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
def createRequest(self, operation, request, outgoingData = None): try: # originatingObject = request.originatingObject() # if originatingObject: # varId = originatingObject.property('RAFT_requestId') # if varId.isValid(): # print('createRequest', '%s->%s' % (str(varId.toString()), str(request.url().toEncoded().data().decode('utf-8')))) # request.setAttribute(QtNetwork.QNetworkRequest.User + 1, varId) # requestId = str(varId.toString()) # if requestId not in self.originalRequestIds: # self.originalRequestIds.append(requestId) # request.setAttribute(QtNetwork.QNetworkRequest.User + 2, varId) url = request.url().toEncoded().data().decode('utf-8') if outgoingData is not None and type(outgoingData) == QIODevice: outgoingData = InterceptFormData(outgoingData) return StoreNetworkReply(self.framework, url, operation, request, outgoingData, self.cookieJar(), QNetworkAccessManager.createRequest(self, operation, request, outgoingData), self) except Exception as error: # exceptions will cause a segfault import traceback print(('--->FIX ME:\n%s' % traceback.format_exc(error))) request.setUrl(QUrl('about:blank')) return QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
def createRequest(self, operation, request, data): url = request.url() #print url if operation == self.GetOperation: actionName = str(url.scheme()) methodName = str(url.encodedHost()) is_action = False if url.scheme() == 'file': is_action = str(url.toEncoded()).endswith(".action") if is_action is False: return QNetworkAccessManager.createRequest( self, operation, request ) if is_action: fileAction = str(url.toEncoded()).rpartition("/")[2].split(".") actionName = fileAction[0] methodName = fileAction[1] result = self.__invokeAction(actionName, methodName, url) if result is not None: return HtmlReply(url, operation, result) return QNetworkAccessManager.createRequest( self, operation, request )
def createRequest(self, op, request, device=None): global lst, pg try: #print(str(request.url())) urlLnk = (request.url()).toString() #print(urlLnk) path = str(request.url().path()) except UnicodeEncodeError: pass #print(path) lower_case = path.lower() #lst = tuple(open("easylist.txt", 'r')) lst = [ "doubleclick.net", "ads", '.jpg', '.png', '.gif', '.css', 'facebook', '.aspx', r"||youtube-nocookie.com/gen_204?", r"youtube.com###watch-branded-actions", "imagemapurl", "b.scorecardresearch.com", "rightstuff.com", "scarywater.net", "popup.js", "banner.htm", "_tribalfusion", "||n4403ad.doubleclick.net^$third-party", ".googlesyndication.com", "graphics.js", "fonts.googleapis.com/css", "s0.2mdn.net", "server.cpmstar.com", "||banzai/banner.$subdocument", "@@||anime-source.com^$document", "/pagead2.", "frugal.gif", "jriver_banner.png", "show_ads.js", '##a[href^="http://billing.frugalusenet.com/"]', "http://jriver.com/video.html", "||animenewsnetwork.com^*.aframe?", "||contextweb.com^$third-party", ".gutter", ".iab", 'http://www.animenewsnetwork.com/assets/[^"]*.jpg' ] block = False for l in lst: if lower_case.find(l) != -1: block = True break if block: #print ("Skipping") #print (request.url().path()) return QNetworkAccessManager.createRequest( self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl())) else: if 'itag=' in urlLnk and 'redirector' not in urlLnk: print('*********') #print() #print(path) #print(urlLnk) #pg.web.setHtml('<html>got the link</html>') #self.lnkUrl.emit(path) f = open('/tmp/AnimeWatch/lnk.txt', 'w') f.write(urlLnk) f.close() return QNetworkAccessManager.createRequest( self, op, request, device) #return QNetworkAccessManager.createRequest(self, op, request, device) #return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl())) else: return QNetworkAccessManager.createRequest( self, op, request, device)
def createRequest(self, op, request, device=None): url = request.url().toString() doFilter = adblockFilter.match(url) if doFilter: try: return QNetworkAccessManager.createRequest(self, self.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl())) except: print("failed to create request ...") return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(nself, op, req, *args, **kwargs): url = req.url().toString() self._logger.debug("Google Translate: %s", url) if '/translate_tts?' not in url: return QNetworkAccessManager.createRequest( nself, op, req, *args, **kwargs) if self._cb and 'consumed' not in self._cb and \ 'ttsspeed=' not in url: # avoid the slower audio playback self._cb['consumed'] = True rep = QNetworkAccessManager.createRequest( nself, op, req, *args, **kwargs) def finished(): try: if not self._cb: pass elif rep.error(): raise RuntimeError("error in network reply") elif rep.header(HEADER_LOCATION): raise RuntimeError("got redirected away") elif rep.header( HEADER_CONTENT_TYPE) != 'audio/mpeg': raise RuntimeError("unexpected Content-Type") elif rep.header(HEADER_CONTENT_LENGTH) < 1024: raise RuntimeError( "Content-Length is too small") else: stream = rep.readAll() if not stream: raise RuntimeError("no stream returned") elif len(stream) < 1024: raise RuntimeError("stream is too small") else: self._cb['okay'](stream) except StandardError as error: try: self._cb['fail'](format(error)) except StandardError: pass finally: try: rep.close() except StandardError: pass rep.finished.connect(finished) return QNetworkAccessManager.createRequest( nself, op, FAKE, *args, **kwargs)
def createRequest(self, operation, request, device): if request.url().scheme() != "gopher": return QNetworkAccessManager.createRequest(self, operation, request, device) if operation == self.GetOperation: # Handle gopher:// URLs separately by creating custom QNetworkReply # objects. reply = GopherReply(request.url()) return reply else: return QNetworkAccessManager.createRequest(self, operation, request, device)
def createRequest(self, op, request, device = None ): path = str(request.url().path()) lower_case = path.lower() lst = ["banner", "ads", r"||youtube-nocookie.com/gen_204?", r"youtube.com###watch-branded-actions", "imagemapurl"] block = False for l in lst: if lower_case.find(l) != -1: block = True break if block: print "Skipping "+request.url().path() return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl())) else: return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(self, operation, request, outgoingData = None): reply = None try: requestUrl = request.url() scheme = str(requestUrl.scheme()) if scheme in ('data', 'about', 'javascript'): reply = QNetworkAccessManager.createRequest(self, operation, request, outgoingData) elif scheme in ('http', 'https'): url = requestUrl.toEncoded().data().decode('utf-8') url_response = None if url in self.request_lookaside: url_response = self.request_lookaside[url] else: responses = [] for row in self.Data.read_responses_by_url(self.cursor, url): responseItems = interface.data_row_to_response_items(row) response_length = str(responseItems[ResponsesTable.RES_LENGTH]) if response_length and int(response_length) > 0 and str(responseItems[ResponsesTable.STATUS]).startswith('2'): responses.append(responseItems) if len(responses) > 0: url_response = responses[-1] # TODO: implement max size limit self.request_lookaside[url] = url_response if not url_response: # try again, with base url if '?' in url: base_url = url[0:url.index('?')] else: base_url = url for row in self.Data.read_responses_starting_with_url(self.cursor, base_url): responseItems = interface.data_row_to_response_items(row) response_length = str(responseItems[ResponsesTable.RES_LENGTH]) if response_length and int(response_length) > 0 and str(responseItems[ResponsesTable.STATUS]).startswith('2'): responses.append(responseItems) if len(responses) > 0: url_response = responses[-1] self.request_lookaside[url] = url_response if url_response: reply = CustomNetworkReply(self, requestUrl, url_response[ResponsesTable.RES_HEADERS], url_response[ResponsesTable.RES_DATA]) except Exception as error: self.framework.report_implementation_error(error) if not reply: # must always return reply request.setUrl(QUrl('about:blank')) reply = QNetworkAccessManager.createRequest(self, operation, request, outgoingData) return reply
def createRequest(self, operation, request, outgoingData=None): try: url = request.url().toEncoded().data().decode('utf-8') if outgoingData is not None and type(outgoingData) == QIODevice: outgoingData = InterceptFormData(outgoingData) return StoreNetworkReply( self.framework, url, operation, request, outgoingData, self.cookieJar(), QNetworkAccessManager.createRequest(self, operation, request, outgoingData), self) except Exception as error: # exceptions will cause a segfault self.framework.report_exception(error) request.setUrl(QUrl('about:blank')) return QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
def createRequest(self, operation, request, data): try: requestFunc = self._dispatcher[request.url().scheme()] except KeyError: self.addHeadersToRequest(request) return QNetworkAccessManager.createRequest(self, operation, request, data) return requestFunc(self, operation, request, data)
def createRequest(self, operation, request, data): method = self.get_method_name(operation) allowed = True req_url = request.url().toString() if req_url.startswith('data:'): req_url = 'data:[%s...]' % req_url[:50] if operation == self.GetOperation: if not self.is_request_allowed(request): request.setUrl(QUrl('forbidden://localhost/')) allowed = False if allowed: if self.proxy().hostName(): suffix = ' via proxy %s:%d' % (self.proxy().hostName(), self.proxy().port()) else: suffix = '' logger.debug('%s [%d] %s%s' % (method.upper(), next(REQ_COUNTER), req_url, suffix)) else: logger_rules.debug('REJECT REQ %s' % req_url) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) #request.setRawHeader('Accept-Encoding', 'gzip') reply = QNetworkAccessManager.createRequest(self, operation, request, data) reply.error.connect(self.handle_network_reply_error) reply.readyRead.connect(lambda x=reply, y=copy(self.traffic_rules):\ handle_reply_ready_read(x, y)) return reply
def createRequest(self, operation, request, data): if request.url().scheme() == "txmt": self.commandUrlRequested.emit(request.url()) elif request.url().scheme() == "tm-file" and operation == self.GetOperation: reply = TmFileReply(self, request.url(), self.GetOperation) return reply return QNetworkAccessManager.createRequest(self, operation, request, data)
def createRequest(self, op, req, outgoingData): do_action("NetworkAccessManagerCreateRequestPre", Bunch(locals())) reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData) if self.m_ignoreSslErrors: reply.ignoreSslErrors() headers = [] for header in req.rawHeaderList(): header = {"name": str(header), "value": str(req.rawHeader(header))} headers.append(header) self.m_idCounter += 1 self.m_ids[reply] = self.m_idCounter data = { "id": self.m_idCounter, "url": req.url().toString(), "method": toString(op), "headers": headers, "time": QDateTime.currentDateTime(), } reply.readyRead.connect(self.handleStarted) do_action("NetworkAccessManagerCreateRequestPost", Bunch(locals())) self.resourceRequested.emit(data) return reply
def createRequest(self, operation, request, data): if operation == self.GetOperation: if not self.is_request_allowed(request): request.setUrl(QUrl('forbidden://localhost/')) else: logger.debug(u'Querying URL: %s' % request.url().toString()) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) reply = QNetworkAccessManager.createRequest(self, operation, request, data) reply.error.connect(self.catch_error) reply = KitNetworkReply(self, reply) # add Base-Url header, then we can get it from QWebView # WTF? if isinstance(request.originatingObject(), QWebFrame): try: reply.setRawHeader( QByteArray('Base-Url'), QByteArray('').append(request.originatingObject().page() .mainFrame().baseUrl().toString())) except Exception as e: logger.debug(e) return reply
def createRequest(self, operation, request, data): if operation == self.GetOperation: if self.is_forbidden(request): # deny GET request for banned media type by setting dummy URL # XXX abort properly request.setUrl(QUrl(QString('forbidden://localhost/'))) else: common.logger.debug( common.to_unicode( request.url().toString().toUtf8().data()).encode( 'utf-8')) #print request.url().toString(), operation request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) reply = QNetworkAccessManager.createRequest(self, operation, request, data) reply.error.connect(self.catch_error) #add Base-Url header, then we can get it from QWebView if isinstance(request.originatingObject(), QWebFrame): try: reply.setRawHeader( QByteArray('Base-Url'), QByteArray('').append(request.originatingObject().page(). mainFrame().baseUrl().toString())) except Exception, e: common.logger.debug(e)
def createRequest(self, op, req, outgoingData): do_action('NetworkAccessManagerCreateRequestPre') reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData) if self.m_ignoreSslErrors: reply.ignoreSslErrors() headers = [] for header in req.rawHeaderList(): header = { 'name': str(header), 'value': str(req.rawHeader(header)) } headers.append(header) self.m_idCounter += 1 self.m_ids[reply] = self.m_idCounter data = { 'id': self.m_idCounter, 'url': req.url().toString(), 'method': self.operationToString(op), 'headers': headers, 'time': QDateTime.currentDateTime() } reply.readyRead.connect(self.handleStarted) do_action('NetworkAccessManagerCreateRequestPost') self.resourceRequested.emit(data) return reply
def createRequest(self, operation, request, post): """Override creating a network request """ url = request.url().toString() if str(request.url().path()).endswith('.ttf'): # block fonts, which can cause webkit to crash common.logger.debug(u'Blocking: {}'.format(url)) request.setUrl(QUrl()) data = post if post is None else post.peek(MAX_POST_SIZE) key = u'{} {}'.format(url, data) use_cache = not url.startswith('file') if self.cache is not None and use_cache and key in self.cache: common.logger.debug(u'Load from cache: {}'.format(key)) content, headers, attributes = self.cache[key] reply = CachedNetworkReply(self, request.url(), content, headers, attributes) else: common.logger.debug(u'Request: {} {}'.format(url, post or '')) reply = QNetworkAccessManager.createRequest( self, operation, request, post) reply.error.connect(self.catch_error) self.active_requests.append(reply) reply.destroyed.connect(self.active_requests.remove) # save reference to original request reply.content = QByteArray() reply.readyRead.connect(self._save_content(reply)) if self.cache is not None and use_cache: reply.finished.connect(self._cache_content(reply, key)) reply.orig_request = request reply.data = self.parse_data(data) return reply
def createRequest(self, op, req, outgoingData): do_action('NetworkAccessManagerCreateRequestPre') reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData) if self.m_ignoreSslErrors: reply.ignoreSslErrors() headers = [] for header in req.rawHeaderList(): header = {'name': str(header), 'value': str(req.rawHeader(header))} headers.append(header) self.m_idCounter += 1 self.m_ids[reply] = self.m_idCounter data = { 'id': self.m_idCounter, 'url': req.url().toString(), 'method': toString(op), 'headers': headers, 'time': QDateTime.currentDateTime() } reply.readyRead.connect(self.handleStarted) do_action('NetworkAccessManagerCreateRequestPost') self.resourceRequested.emit(data) return reply
def createRequest(self, op, req, outgoingData): if op == QNetworkAccessManager.GetOperation: qDebug('HTTP/1.1 GET Request') elif op == QNetworkAccessManager.PostOperation: qDebug('HTTP/1.1 POST Request') elif op == QNetworkAccessManager.HeadOperation: qDebug('HTTP/1.1 HEAD Request') elif op == QNetworkAccessManager.PutOperation: qDebug('HTTP/1.1 PUT Request') elif op == QNetworkAccessManager.DeleteOperation: qDebug('HTTP/1.1 DELETE Request') elif op == QNetworkAccessManager.CustomOperation: qDebug('HTTP/1.1 CUSTOM Request') else: qWarning('Unexpected HTTP Operation Type') qDebug('URL %s' % req.url().toString()) do_action('NetworkAccessManagerCreateRequestPre', Bunch(locals())) reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData) if self.m_ignoreSslErrors == 'yes': reply.ignoreSslErrors() do_action('NetworkAccessManagerCreateRequestPost', Bunch(locals())) return reply
def createRequest(self, operation, request, data): if operation == self.GetOperation: if not self.is_request_allowed(request): request.setUrl(QUrl('forbidden://localhost/')) else: logger.debug(u'Quering URL: %s' % request.url().toString()) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) reply = QNetworkAccessManager.createRequest(self, operation, request, data) reply.error.connect(self.catch_error) reply = KitNetworkReply(self, reply) # add Base-Url header, then we can get it from QWebView # WTF? if isinstance(request.originatingObject(), QWebFrame): try: reply.setRawHeader( QByteArray('Base-Url'), QByteArray('').append(request.originatingObject().page(). mainFrame().baseUrl().toString())) except Exception as e: logger.debug(e) return reply
def createRequest(self, operation, request, device): path = o = request.url().toString() if path.startswith('app://') or path.startswith('lens://'): if path == 'app:///': path = 'file://' + self._uri_app_base + 'app.html' logger.debug('Loading app resource: {0} ({1})'.format(o, path)) elif path.startswith('app://'): path = path.replace('app://', 'file://' + self._uri_app_base) logger.debug('Loading app resource: {0} ({1})'.format(o, path)) # variable substitution path = path.replace('$backend', 'qt4') elif path.startswith('lens://'): path = path.replace('lens://', 'file://' + self._uri_lens_base) # make lens.css backend specific path = path.replace('lens.css', 'lens-qt4.css') logger.debug('Loading lens resource: {0} ({1})'.format(o, path)) request.setUrl(QUrl(QString(path))) return QNetworkAccessManager.createRequest(self, operation, request, device)
def createRequest(self, operation, request, device): path = o = request.url().toString() if path.startswith('app://') or path.startswith('lens://'): if path == 'app:///': path = 'file://' + self._uri_app_base + 'app.html' logger.debug('Loading app resource: {0} ({1})'.format(o, path)) elif path.startswith('app://'): path = path.replace('app://', 'file://' + self._uri_app_base) logger.debug('Loading app resource: {0} ({1})'.format(o, path)) # variable substitution path = path.replace('$backend', 'qt4') elif path.startswith('lens://'): path = path.replace('lens://', 'file://' + self._uri_lens_base) # make lens.css backend specific path = path.replace('lens.css', 'lens-qt4.css') logger.debug('Loading lens resource: {0} ({1})'.format( o, path)) request.setUrl(QUrl(QString(path))) return QNetworkAccessManager.createRequest(self, operation, request, device)
def createRequest(self, operation, request, data): #print("mymanager handles ", request.url()) if self.referer and self.webpage.history().canGoBack(): request.setRawHeader('Referer', self.referer) for key in self.header.keys(): request.setRawHeader(key, self.header[key]) return QNetworkAccessManager.createRequest(self, operation, request, data)
def createRequest(self, op, request, outgoingData=None): """ Protected method to create a request. @param op the operation to be performed (QNetworkAccessManager.Operation) @param request reference to the request object (QNetworkRequest) @param outgoingData reference to an IODevice containing data to be sent (QIODevice) @return reference to the created reply object (QNetworkReply) """ scheme = request.url().scheme() if scheme == "https" and (not SSL_AVAILABLE or not QSslSocket.supportsSsl()): return NetworkProtocolUnknownErrorReply(scheme, self) if op == QNetworkAccessManager.PostOperation and outgoingData is not None: outgoingDataByteArray = outgoingData.peek(1024 * 1024) Helpviewer.HelpWindow.HelpWindow.passwordManager().post(request, outgoingDataByteArray) reply = None if unicode(scheme) in self.__schemeHandlers: reply = self.__schemeHandlers[unicode(scheme)].createRequest(op, request, outgoingData) if reply is not None: return reply if not self.__acceptLanguage.isEmpty(): req = QNetworkRequest(request) req.setRawHeader("Accept-Language", self.__acceptLanguage) else: req = request # AdBlock code if op == QNetworkAccessManager.GetOperation: if self.__adblockNetwork is None: self.__adblockNetwork = Helpviewer.HelpWindow.HelpWindow.adblockManager().network() reply = self.__adblockNetwork.block(req) if reply is not None: reply.setParent(self) return reply # set cache policy if op == QNetworkAccessManager.GetOperation: urlHost = req.url().host() for host in self.NoCacheHosts: if host in urlHost: req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork) break else: req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork) reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData) self.emit( SIGNAL("requestCreated(QNetworkAccessManager::Operation, const QNetworkRequest&, QNetworkReply*)"), op, req, reply, ) return reply
def createRequest(self, operation, request, data): if operation == self.GetOperation: if request.url().scheme() == self.url_scheme: # handle epwing:// urls separately by creating custom QNetworkReply objects. reply = EpwingReply(self, request, request.url(), self.GetOperation, self.book_manager) return reply #return self.old_manager.createRequest(operation, request, data) return QNetworkAccessManager.createRequest(self, operation, request, data)
def createRequest(self, op, request, device = None ): global lst try: path = str(request.url().path()) except UnicodeEncodeError: pass lower_case = path.lower() #lst = tuple(open("easylist.txt", 'r')) lst = ["doubleclick.net" ,"ads",'.jpg','redirector','itag=','.png','.gif','.css','google','facebook','.aspx',r"http[^'].mp4",r"http[^'].flv", r"||youtube-nocookie.com/gen_204?", r"youtube.com###watch-branded-actions", "imagemapurl","b.scorecardresearch.com","rightstuff.com","scarywater.net","popup.js","banner.htm","_tribalfusion","||n4403ad.doubleclick.net^$third-party",".googlesyndication.com","graphics.js","fonts.googleapis.com/css","s0.2mdn.net","server.cpmstar.com","||banzai/banner.$subdocument","@@||anime-source.com^$document","google","/pagead2.","frugal.gif","jriver_banner.png","show_ads.js",'##a[href^="http://billing.frugalusenet.com/"]',"http://jriver.com/video.html","||animenewsnetwork.com^*.aframe?","||contextweb.com^$third-party",".gutter",".iab",'http://www.animenewsnetwork.com/assets/[^"]*.jpg'] block = False for l in lst: if lower_case.find(l) != -1: block = True break if block: print ("Skipping") print (request.url().path()) return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl())) else: return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(self, operation, request, outgoingData=None): try: url = request.url().toEncoded().data().decode("utf-8") if outgoingData is not None and type(outgoingData) == QIODevice: outgoingData = InterceptFormData(outgoingData) return StoreNetworkReply( self.framework, url, operation, request, outgoingData, self.cookieJar(), QNetworkAccessManager.createRequest(self, operation, request, outgoingData), self, ) except Exception as error: # exceptions will cause a segfault self.framework.report_exception(error) request.setUrl(QUrl("about:blank")) return QNetworkAccessManager.createRequest(self, operation, request, outgoingData)
def createRequest(self, op, request, device): # prefer valid cache instead of network self.reqCount = self.reqCount + 1 if re.match(".*?pubads\.g\.doubleclick\.net*", request.url().toString()): request.setUrl(QUrl("http://img6a.flixcart.com/www/prod/images/flipkart_india-e5f5aa9f.png")) # print "REQ :",self.reqCount," ",request.url() request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) reply = QNetworkAccessManager.createRequest(self, op, request, device) reply.error.connect(self.logNetworkErrors) return reply
def createRequest(self, op, request, device=None): global block_list try: path = str(request.url().toString()) except UnicodeEncodeError: path = str(request.url().path()) pass lower_path = path.lower() #lst = tuple(open("easylist.txt", 'r')) block_list = [ "doubleclick.net", "ads", r"||youtube-nocookie.com/gen_204?", r"youtube.com###watch-branded-actions", "imagemapurl", "b.scorecardresearch.com", "rightstuff.com", "scarywater.net", "popup.js", "banner.htm", "_tribalfusion", "||n4403ad.doubleclick.net^$third-party", ".googlesyndication.com", "graphics.js", "fonts.googleapis.com/css", "s0.2mdn.net", "server.cpmstar.com", "||banzai/banner.$subdocument", "@@||anime-source.com^$document", "/pagead2.", "frugal.gif", "jriver_banner.png", "show_ads.js", '##a[href^="http://billing.frugalusenet.com/"]', "http://jriver.com/video.html", "||animenewsnetwork.com^*.aframe?", "||contextweb.com^$third-party", ".gutter", ".iab", 'http://www.animenewsnetwork.com/assets/[^"]*.jpg', 'revcontent' ] block = False for l in block_list: if l in lower_path: block = True break if block: print("Skipping") print(path) return QNetworkAccessManager.createRequest( self, QNetworkAccessManager.GetOperation, QtNetwork.QNetworkRequest(QtCore.QUrl())) else: return QNetworkAccessManager.createRequest(self, op, request, device)
def _manager_create_request(self, operation, request, data): url = unicode(request.url().toString()) operation_name = self._operation_names[operation].upper() self._debug(INFO, "Request: %s %s" % (operation_name, url)) for h in request.rawHeaderList(): self._debug(DEBUG, " %s: %s" % (h, request.rawHeader(h))) if self._url_filter: if self._url_filter(self._operation_names[operation], url) is False: self._debug(INFO, "URL filtered: %s" % url) request.setUrl(QUrl("about:blank")) else: self._debug(DEBUG, "URL not filtered: %s" % url) reply = QNetworkAccessManager.createRequest(self.manager, operation, request, data) return reply
def createRequest(self, op, request, device = None): if InjectedQNetworkRequest.thatRequestHasMagicQt4(request): r = InjectedNetworkReply(self, request.url(), self.response.read(), op) default_cookie_domain = self._cookie_default_domain(request) cookiejar = self._createCookieJarfromInjectedResponse(default_cookie_domain) self.setCookieJar(cookiejar) else: r = QNetworkAccessManager.createRequest(self, op, request, device) r.finished.connect(self.checkAutoCloseUrls) return r
def createRequest(self, op, request, device = None): if InjectedQNetworkRequest.thatRequestHasMagicQt4(request): r = InjectedNetworkReply(self, request.url(), self.urllib_response.read(), op, self.urllib_request, self.urllib_response) default_cookie_domain = self._cookie_default_domain(request) cookiejar = self._import_cookie_jar(self.http_cookie_jar, default_cookie_domain) self.setCookieJar(cookiejar) else: self.urllib_response = None self.urllib_request = None original_r = QNetworkAccessManager.createRequest(self, op, request, device) original_r.sslErrors.connect(self.sslErrorHandler) r = SniffingNetworkReply(self, request, original_r, op) r.finished.connect(self.requestFinishedActions) return r
def createRequest(self, op, request, outgoingData): if op == QNetworkAccessManager.GetOperation: try: urlPath = unicode(request.url().path()).encode( 'UTF-8', 'replace') if urlPath.rfind('.js') != -1 and urlPath.rfind( 'assets') == -1: reply = ReportJSRenderReply(self, request) return reply elif request.url().scheme() == "dff-node-thumbnail": reply = NodeThumbnailRenderReply(self, request) return reply except Exception as e: print 'UrlRenderer error: ' + str(e) request = QNetworkAccessManager.createRequest(self, op, request, outgoingData) return request
def createRequest(self, op, request, outgoingData = None): """ Protected method to create a request. @param op the operation to be performed (QNetworkAccessManager.Operation) @param request reference to the request object (QNetworkRequest) @param outgoingData reference to an IODevice containing data to be sent (QIODevice) @return reference to the created reply object (QNetworkReply) """ if self.primaryManager is not None and \ self.__webPage is not None: pageRequest = QNetworkRequest(request) self.__webPage.populateNetworkRequest(pageRequest) return self.primaryManager.createRequest(op, pageRequest, outgoingData) else: return QNetworkAccessManager.createRequest(self, op, request, outgoingData)
def createRequest(self, op, request, device): # prefer valid cache instead of network self.reqCount = self.reqCount + 1 if re.match(".*?pubads\.g\.doubleclick\.net*", request.url().toString()): request.setUrl( QUrl( "http://img6a.flixcart.com/www/prod/images/flipkart_india-e5f5aa9f.png" )) # print "REQ :",self.reqCount," ",request.url() request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) reply = QNetworkAccessManager.createRequest(self, op, request, device) reply.error.connect(self.logNetworkErrors) return reply
def createRequest(self, op, req, outgoingData): if op == QNetworkAccessManager.GetOperation: qDebug('HTTP/1.1 GET Request') elif op == QNetworkAccessManager.PostOperation: qDebug('HTTP/1.1 POST Request') elif op == QNetworkAccessManager.HeadOperation: qDebug('HTTP/1.1 HEAD Request') elif op == QNetworkAccessManager.PutOperation: qDebug('HTTP/1.1 PUT Request') elif op == QNetworkAccessManager.DeleteOperation: qDebug('HTTP/1.1 DELETE Request') elif op == QNetworkAccessManager.CustomOperation: qDebug('HTTP/1.1 CUSTOM Request') else: qWarning('Unexpected HTTP Operation Type') qDebug('URL %s' % req.url().toString()) return QNetworkAccessManager.createRequest(self, op, req, outgoingData)
def createRequest(self, operation, request, data): if operation == self.GetOperation: if self.is_forbidden(request): # deny GET request for banned media type by setting dummy URL # XXX abort properly request.setUrl(QUrl('forbidden://localhost/')) else: common.logger.debug(common.to_unicode(request.url().toString()).encode('utf-8')) #print request.url().toString(), operation request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) reply = QNetworkAccessManager.createRequest(self, operation, request, data) reply.error.connect(self.catch_error) # add Base-Url header, then we can get it from QWebView if isinstance(request.originatingObject(), QWebFrame): try: reply.setRawHeader(QByteArray('Base-Url'), QByteArray('').append(request.originatingObject().page().mainFrame().baseUrl().toString())) except Exception, e: common.logger.debug(e)
def createRequest(self, operation, request, data): if operation == self.GetOperation: if self.is_forbidden(request): # deny GET request for banned media type by setting dummy URL request.setUrl(QUrl(QString('forbidden://localhost/'))) else: print request.url().toString() else: pass #print 'POST' #print request.url().toString() #data.open(QIODevice.ReadOnly) #print data.readAll() #print data.peek(100000000000) #data.seek(0) #data.close() request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) reply = QNetworkAccessManager.createRequest(self, operation, request, data) #reply.finished.connect(self.catch_finished) reply.error.connect(self.catch_error) return reply
def createRequest(self, operation, request, data): #request.setRawHeader("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7"); #print 'createRequest: url is '+request.url() #print 'mymanager handles ',request.url() #file = open('request.html', 'a') #file.write(request.url().toString()+'\n======================================================================\n\n\n\n\n\n\n') #file.close() request.setRawHeader("Connection","keep-alive"); request.setRawHeader("Host", "google.com") request.setRawHeader("Proxy-Connection", "keep-alive") request.setRawHeader("Cache-Control", "max-age=0") request.setRawHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") request.setRawHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36") request.setRawHeader("DNT", "1") request.setRawHeader("Referer", "http://www.google.com") request.setRawHeader("Accept-Encoding", "deflate, sdch") request.setRawHeader("Accept-Language", "en-US,en;q=0.8") request.setRawHeader("Cookie", "") #a = request.rawHeader("Cookie") #print a sys.stdout.write('.') # progress bar return QNetworkAccessManager.createRequest( self, operation, request, data )
def create_request(self, operation, request, data): """ Reimplemented to intercept requests. Stops blacklisted requests, matches requests with replies. Stores on a PostgreSQL database. """ if es_url_local(request.url()) and not es_font(request.url()): return QNetworkAccessManager.createRequest( self, operation, request, data) if (usando_whitelist(self.whitelist, request.url()) or es_font(request.url())): print "FILTERING %s" % request.url().toString()[:255] return QNetworkAccessManager.createRequest( self, operation, QNetworkRequest(QUrl("about:blank")), data) request.setAttribute( QNetworkRequest.HttpPipeliningAllowedAttribute, True) response = QNetworkAccessManager.createRequest( self, operation, request, data) if operation == QNetworkAccessManager.PostOperation: post_str = unicode(data.peek(4096)) data_json = filtra(parse_qsl(post_str, keep_blank_values=True)) else: data_json = None def indice(reply, idx): """ This function returns a closure enclosing the current index and its associated reply. This is required since there's no knowing when (if at all) a request will be replied to; once it happens, the index will surely have changed. The 'cheatgc' list is required because of a bug where the reference for the 'reply' and 'idx' objects is lost sometimes after 'ret' completes, despite having created a closure referencing them. By appending to the list, garbage collection is prevented. """ # please don't do garbage collection... self.cheatgc.append(reply) self.cheatgc.append(idx) def ret(): """ Closure; freezes 'reply' and 'idx' so they will be accessible at the time the request finalizes and this closure is called. """ try: encabezados = filtra(reply.rawHeaderPairs()) (statuscode, _) = reply.attribute( QNetworkRequest.HttpStatusCodeAttribute).toInt() if self.log: self.log.store_reply({ "id": self.instance_id, "idx": idx, "scheme": unicode(reply.url().scheme()), "host": unicode(reply.url().host()), "path": unicode(reply.url().path()), "query": filtra(reply.url().encodedQueryItems()), "fragment": notnull( unicode(reply.url().fragment())), "status": statuscode, "headers": encabezados }) # ...until we're done with the request # (pyqt/sip related trouble) self.cheatgc.remove(reply) self.cheatgc.remove(idx) #print "[%s]" % (len(self.cheatgc)) except NameError as name_error: print name_error print "Except NameError!" return ret response.finished.connect(indice(response, self.count)) if self.log: headers = [] for header in request.rawHeaderList(): headers.append((header, request.rawHeader(header))) self.log.store_request({ "id": self.instance_id, "idx": self.count, "op": OPERATIONS[operation], "scheme": unicode(request.url().scheme()), "host": unicode(request.url().host()), "path": unicode(request.url().path()), "query": filtra(request.url().encodedQueryItems()), "fragment": notnull( unicode(request.url().fragment())), "data": data_json, "source": unicode( request.originatingObject().requestedUrl().host()), "headers": filtra(headers) }) self.count += 1 return response
def createRequest(self, op, req, outgoingData): for k in self.args: req.setRawHeader(k, self.args[k] ) reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData) return reply
def createRequest(self, op, request, device=None): request.setRawHeader( 'User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36' ) return QNetworkAccessManager.createRequest(self, op, request, device)
def createRequest(self, operation, request, data): if request.url().scheme() != "tentia": return QNetworkAccessManager.createRequest(self, operation, request, data) else: self.tentia_callback(request.url()) return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QNetworkRequest(QtCore.QUrl()))
def create_request(self, operation, request, data): """ Reimplemented to intercept requests. Stops blacklisted requests, matches requests with replies. Stores on a PostgreSQL database. """ if es_url_local(request.url()) and not es_font(request.url()): return QNetworkAccessManager.createRequest(self, operation, request, data) if (usando_whitelist(self.whitelist, request.url()) or es_font(request.url())): print "FILTERING %s" % request.url().toString()[:255] return QNetworkAccessManager.createRequest( self, operation, QNetworkRequest(QUrl("about:blank")), data) request.setAttribute(QNetworkRequest.HttpPipeliningAllowedAttribute, True) response = QNetworkAccessManager.createRequest(self, operation, request, data) if operation == QNetworkAccessManager.PostOperation: post_str = unicode(data.peek(4096)) data_json = filtra(parse_qsl(post_str, keep_blank_values=True)) else: data_json = None def indice(reply, idx): """ This function returns a closure enclosing the current index and its associated reply. This is required since there's no knowing when (if at all) a request will be replied to; once it happens, the index will surely have changed. The 'cheatgc' list is required because of a bug where the reference for the 'reply' and 'idx' objects is lost sometimes after 'ret' completes, despite having created a closure referencing them. By appending to the list, garbage collection is prevented. """ # please don't do garbage collection... self.cheatgc.append(reply) self.cheatgc.append(idx) def ret(): """ Closure; freezes 'reply' and 'idx' so they will be accessible at the time the request finalizes and this closure is called. """ try: encabezados = filtra(reply.rawHeaderPairs()) (statuscode, _) = reply.attribute( QNetworkRequest.HttpStatusCodeAttribute).toInt() if self.log: self.log.store_reply({ "id": self.instance_id, "idx": idx, "scheme": unicode(reply.url().scheme()), "host": unicode(reply.url().host()), "path": unicode(reply.url().path()), "query": filtra(reply.url().encodedQueryItems()), "fragment": notnull(unicode(reply.url().fragment())), "status": statuscode, "headers": encabezados }) # ...until we're done with the request # (pyqt/sip related trouble) self.cheatgc.remove(reply) self.cheatgc.remove(idx) #print "[%s]" % (len(self.cheatgc)) except NameError as name_error: print name_error print "Except NameError!" return ret response.finished.connect(indice(response, self.count)) if self.log: headers = [] for header in request.rawHeaderList(): headers.append((header, request.rawHeader(header))) self.log.store_request({ "id": self.instance_id, "idx": self.count, "op": OPERATIONS[operation], "scheme": unicode(request.url().scheme()), "host": unicode(request.url().host()), "path": unicode(request.url().path()), "query": filtra(request.url().encodedQueryItems()), "fragment": notnull(unicode(request.url().fragment())), "data": data_json, "source": unicode(request.originatingObject().requestedUrl().host()), "headers": filtra(headers) }) self.count += 1 return response
def createRequest(self, operation, request, outgoingData=None): reply = None try: requestUrl = request.url() scheme = str(requestUrl.scheme()) if scheme in ('data', 'about', 'javascript'): reply = QNetworkAccessManager.createRequest( self, operation, request, outgoingData) elif scheme in ('http', 'https'): url = requestUrl.toEncoded().data().decode('utf-8') url_response = None if url in self.request_lookaside: url_response = self.request_lookaside[url] else: responses = [] for row in self.Data.read_responses_by_url( self.cursor, url): responseItems = interface.data_row_to_response_items( row) response_length = str( responseItems[ResponsesTable.RES_LENGTH]) if response_length and int( response_length) > 0 and str(responseItems[ ResponsesTable.STATUS]).startswith('2'): responses.append(responseItems) if len(responses) > 0: url_response = responses[-1] # TODO: implement max size limit self.request_lookaside[url] = url_response if not url_response: # try again, with base url if '?' in url: base_url = url[0:url.index('?')] else: base_url = url for row in self.Data.read_responses_starting_with_url( self.cursor, base_url): responseItems = interface.data_row_to_response_items( row) response_length = str( responseItems[ResponsesTable.RES_LENGTH]) if response_length and int( response_length) > 0 and str( responseItems[ResponsesTable. STATUS]).startswith('2'): responses.append(responseItems) if len(responses) > 0: url_response = responses[-1] self.request_lookaside[url] = url_response if url_response: reply = CustomNetworkReply( self, requestUrl, url_response[ResponsesTable.RES_HEADERS], url_response[ResponsesTable.RES_DATA]) except Exception as error: self.framework.report_implementation_error(error) if not reply: # must always return reply request.setUrl(QUrl('about:blank')) reply = QNetworkAccessManager.createRequest( self, operation, request, outgoingData) return reply
def createRequest(self,op,request,outgoingdata): scheme=request.url().scheme() # print unicode(request.url().path()) if scheme==QLatin1String('ms-its'): return PyChmNetReply(request,request.url(),self.qwebview,self.qwebview) return QNetworkAccessManager.createRequest(self,op,request,outgoingdata)
def createRequest(self, operation, request, data): if request.url().scheme() != "bungloo": return QNetworkAccessManager.createRequest(self, operation, request, data) else: self.bungloo_callback(request.url()) return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QNetworkRequest(QtCore.QUrl()))