def _import_cookie_jar(self,http_cookie_jar,default_domain): cj = QNetworkCookieJar() cookie_attrs = http_cookie_jar.http_header_attrs(self.urllib_request) cookies = self._parse_cookie_attribs_into_QtCookies_list(cookie_attrs, default_domain) cj.setAllCookies(cookies) return cj
def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600)): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) self.page = GhostWebPage(Ghost._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) self.set_viewport_size(*viewport_size) self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show()
class ShoutCastForm(PluginBase.PluginBase): '''Grab Shoutcast streams and save them as "bookmarks" - and play them on the currently selected server. General shoutcast information is not preserved between runs. Also, the shoutcast server/API is pretty lame so timeouts actually occur quite frequently. ''' moduleName = '&Shoutcast' moduleIcon = "network-workgroup" def load(self): pass def event(self, event): if event.type() == QEvent.Paint: if not hasattr(self, 'webView'): self._load() self.event = super(ShoutCastForm, self).event return False def _load(self): self.cookie = QNetworkCookie('Settings', 'Player~others|Bandwidth~ALL|Codec~ALL') self.cookie.setDomain('.shoutcast.com') self.cookie.setExpirationDate(QDateTime()) self.cookie.setPath('/') self.webView = QWebView(self) self.webPage = self.webView.page() self.cookiejar = QNetworkCookieJar() self.cookiejar.setAllCookies([self.cookie]) self.webPage.networkAccessManager().setCookieJar(self.cookiejar) self.layout = QVBoxLayout(self) self.layout.addWidget(self.webView) self.webView.load(QUrl(HOMEURL)) self.webPage.setLinkDelegationPolicy(QWebPage.DelegateExternalLinks) self.connect(self.webPage, SIGNAL('linkClicked(const QUrl&)'), self._processLink) def _processLink(self, url): if url.host() == TUNEIN: self._playStation(unicode(url.toString())) else: self.webView.load(url) self.webView.show() def _playStation(self, url): try: streamList = streamTools.getStreamList(url) except streamTools.ParseError: return if streamList: self.modelManager.playQueue.extend(streamList)
def load_cookies(): '''unpickle the cookie jar''' jar = QNetworkCookieJar() try: with open(join(homedir(), COOKIE_FILE)) as file: cookies = [] for name, value in pickle.load(file): cookies.append(QNetworkCookie(name, value)) jar.setCookiesFromUrl(cookies, QUrl(COOKIE_URL)) except IOError: pass # no cookie file is okay return jar
def load_cookies(self, url, cookie_strings): p = urlparse(url) cookies = list() for c in cookie_strings: n = c.find('=') name, value = c[:n], c[n+1:] cookie = QNetworkCookie(name, value) cookie.setDomain(p.netloc) cookies.append(cookie) cookiejar = QNetworkCookieJar() cookiejar.setAllCookies(cookies) self.networkAccessManager().setCookieJar(cookiejar)
def __init__(self, parent = None): """ Constructor @param parent reference to the parent object (QObject) """ QNetworkCookieJar.__init__(self, parent) self.__loaded = False self.__acceptCookies = self.AcceptOnlyFromSitesNavigatedTo self.__saveTimer = AutoSaver(self, self.save) self.__cookiesFile = os.path.join(Utilities.getConfigDir(), "browser", "cookies.ini")
def _createCookieJarfromInjectedResponse(self, default_domain): #ugly, but works around bugs in parseCookies cookies = [] for cookie_header in self._getCookieHeaders(): tmp_cookieList = QNetworkCookie.parseCookies(cookie_header) tmp_cookie = tmp_cookieList[0] if not tmp_cookie.domain(): tmp_cookie.setDomain(QString(default_domain)) cookies = cookies + tmp_cookieList cj = QNetworkCookieJar() cj.setAllCookies(cookies) return cj
def __init__(self, my_cookie_dict): super(QWebView, self).__init__() # 将字典转化成QNetworkCookieJar的格式 self.cookie_jar = QNetworkCookieJar() cookies = [] for key, values in my_cookie_dict.items(): my_cookie = QNetworkCookie(QByteArray(key), QByteArray(values)) #my_cookie.setDomain('.baidu.com') cookies.append(my_cookie) self.cookie_jar.setAllCookies(cookies) # 如果没有在前面设置domain,那么可以在这里指定一个url作为domain # self.cookie_jar.setCookiesFromUrl(cookies, QUrl('https://www.baidu.com/')) # 最后cookiejar替换完成 self.page().networkAccessManager().setCookieJar(self.cookie_jar)
def setCookiesFromUrl(self, cookieList, url): if self.__read_only: return True resp = QNetworkCookieJar.setCookiesFromUrl(self, cookieList, url) if resp: self.emit(SIGNAL('cookieJarUpdated()')) return resp
def create_page(self, wait_timeout=20, wait_callback=None, is_popup=False, max_resource_queued=None, download_images=True, prevent_download=[]): """Create a new GhostWebPage :param wait_timeout: The timeout used when we want to load a new url. :param wait_callback: An optional callable that is periodically executed until Ghost stops waiting. :param is_popup: Indicates if the QWebPage it's a popup :param max_resource_queued: Indicates witch it's the max number of resources that can be saved in memory. If None then no limits are applied. If 0 then no resources are kept. If the number it's > 0 then the number of resources won't be more than max_resource_queued :param download_images: Indicate if the browser download or not the images :param prevent_download: A List of extensions of the files that you want to prevent from downloading """ cache_name = self.cache_dir if self.share_cache else self.cache_dir + str( random.randint(0, 100000000)) network_manager = NetworkAccessManager( cache_dir=cache_name, cache_size=self.cache_size, prevent_download=prevent_download) if not self.share_cookies or len(self.network_managers) == 0: cookie_jar = QNetworkCookieJar() network_manager.setCookieJar(cookie_jar) self.cookie_jar = cookie_jar else: network_manager.setCookieJar(self.cookie_jar) if self.share_cookies: self.cookie_jar.setParent(None) self.network_managers.append(network_manager) page = GhostWebPage(app=Ghost._app, network_manager=network_manager, wait_timeout=wait_timeout, wait_callback=wait_callback, viewport_size=self.viewport_size, user_agent=self.user_agent, log_level=self.log_level, create_page_callback=self.create_page, is_popup=is_popup, max_resource_queued=max_resource_queued, download_images=download_images) page.removeWindowFromList.connect(self._remove_page) self._pages.append(page) if self.display: self.webview = QtWebKit.QWebView() self.webview.setPage(page) self.webview.show() return page, (len(self._pages) - 1)
def initProperty(): AppProperty.AppTitle = "OSChina" AppProperty.AppIcon = QIcon("imgs/icon.png") AppProperty.HomeDir = getcwd() AppProperty._service_ = Service() AppProperty._notifications_ = NotificationPresenter() AppProperty.CookieJar = QNetworkCookieJar()
def initProperty(): manifest = json.load(file('./view/app.json')) app.AppTitle = manifest['name'] if manifest['name'] else app.AppTitle app.AppUser = manifest['user'] if manifest['user'] else app.AppUser app.Debug = manifest['debug'] if manifest['debug'] else app.Debug app.HomeDir = getcwd() app.Template = manifest['win'] if manifest['win'] else app.Template app.App = manifest['app'] if manifest['app'] else app.App app.ViewDir = app.HomeDir+'/view/'+app.Template+'/' app.AppDir = app.HomeDir+'/view/'+app.App+'/' app.AppIcon = QIcon(app.ViewDir + (manifest['ico'] if manifest['ico'] else app.AppIcon)) #主页面配置 app.MainSrc = manifest['main']['url'] if manifest['main']['url'] else app.MainSrc app.WinWidth = manifest['main']['width'] if manifest['main']['width'] else app.WinWidth app.WinHeight = manifest['main']['height'] if manifest['main']['height'] else app.WinHeight app.Opacity = manifest['opacity'] if manifest['opacity'] else app.Opacity #对话框配置 app.DialogSrc = manifest['dialog']['url'] if manifest['dialog']['url'] else app.DialogSrc app.DialogCon = manifest['dialog']['content'] if manifest['dialog']['content'] else app.DialogCon app.DialogWidth = manifest['dialog']['width'] if manifest['dialog']['width'] else app.DialogWidth app.DialogHeight = manifest['dialog']['height'] if manifest['dialog']['height'] else app.DialogHeight app.DialogOpacity = manifest['dialog']['opacity'] if manifest['dialog']['opacity'] else app.DialogOpacity app.Music = mp3play.load('./view/winView/assets/notify.mp3') # 各类初始化 app._notifications_ = NotificationPresenter() app.CookieJar = QNetworkCookieJar() app.MainWin = Window(app.MainSrc, app.WinWidth, app.WinHeight) # 打开窗口 app._tool_ = toolKit.ToolKit(app.MainWin) app._robots_ = robots()
def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), cache_dir='/tmp/ghost.py'): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) self.page = GhostWebPage(Ghost._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) self.set_viewport_size(*viewport_size) self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show()
def __init__(self, supportUrl, parent=None): super(GeniusBarReserser, self).__init__(parent) self.ui = Ui_Dialog() self.ui.setupUi(self) self.supportUrl = supportUrl # self.ui.webView.load(QUrl('http://www.apple.com/cn/retail/shanghaiiapm/')) self.netWorkmgr = QNetworkAccessManager(self) cookieJar = QNetworkCookieJar() self.netWrokMgr.setCookieJar(cookieJar)
def _load(self): self.cookie = QNetworkCookie('Settings', 'Player~others|Bandwidth~ALL|Codec~ALL') self.cookie.setDomain('.shoutcast.com') self.cookie.setExpirationDate(QDateTime()) self.cookie.setPath('/') self.webView = QWebView(self) self.webPage = self.webView.page() self.cookiejar = QNetworkCookieJar() self.cookiejar.setAllCookies([self.cookie]) self.webPage.networkAccessManager().setCookieJar(self.cookiejar) self.layout = QVBoxLayout(self) self.layout.addWidget(self.webView) self.webView.load(QUrl(HOMEURL)) self.webPage.setLinkDelegationPolicy(QWebPage.DelegateExternalLinks) self.connect(self.webPage, SIGNAL('linkClicked(const QUrl&)'), self._processLink)
def __init__(self, gui=False): if not Kit._app: Kit._app = QApplication([]) manager = KitNetworkAccessManager() manager.finished.connect(self.network_reply_handler) self.cookie_jar = QNetworkCookieJar() manager.setCookieJar(self.cookie_jar) self.page = KitPage() self.page.setNetworkAccessManager(manager) self.view = KitWebView() self.view.setPage(self.page) self.view.setApplication(Kit._app) if gui: self.view.show()
def set_cookies_from_url(self, cookies, url): """ Reimplementation from base class. Prevents cookies from being set if not from whitelisted domains. """ if ".".join(unicode(url.host()).split('.')[-2:]) not in self.allowed: ret = [] else: ret = cookies if ret: log(format_cookie(url, ret)) return QNetworkCookieJar.setCookiesFromUrl(self, ret, url)
def _MozillaCookieJar_to_QnetworkCookieJar(cookie_file): def line2qcookie(line): domain,initial_dot,path,isSecure,expires,name,value=line.split() isSecure=(isSecure=="TRUE") dt=QDateTime() dt.setTime_t(int(expires)) expires=dt c=QNetworkCookie() c.setDomain(domain) c.setPath(path) c.setSecure(isSecure) c.setExpirationDate(expires) c.setName(name) c.setValue(value) return c cj=QNetworkCookieJar() if os.path.exists(cookie_file): cj.setAllCookies([line2qcookie(line) for line in open(cookie_file) if not line.startswith("#") and line and not line.isspace()]) #~ print cj.allCookies() else: raise IOError("File not found: %s"%cookie_file) return cj
def cookiesForUrl(self, url): """ Public method to get the cookies for a URL. @param url URL to get cookies for (QUrl) @return list of cookies (list of QNetworkCookie) """ if not self.__loaded: self.load() globalSettings = QWebSettings.globalSettings() if globalSettings.testAttribute(QWebSettings.PrivateBrowsingEnabled): return [] return QNetworkCookieJar.cookiesForUrl(self, url)
def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800,600)): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.loaded = True if not 'DISPLAY' in os.environ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display self.app = QApplication(['ghost']) self.page = GhostWebPage(self.app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0); self.set_viewport_size(*viewport_size) self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show()
def load_cookies(self, cookie_storage, keep_old=False): """load from cookielib's CookieJar or Set-Cookie3 format text file. :param cookie_storage: file location string on disk or CookieJar instance. :param keep_old: Don't reset, keep cookies not overridden. """ cookie_jar = QNetworkCookieJar() if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) cj.load() self.toQtCookieJar(cj, cookie_jar) elif cookie_storage.__class__.__name__.endswith('CookieJar'): self.toQtCookieJar(cookie_storage, cookie_jar, keep_old) else: raise ValueError('unsupported cookie_storage type.') return cookie_jar
def __init__(self, gui=False, traffic_rules=None): if not WKitScope.app: raise InternalError('You should use Browser instance' ' inside `with WKitScope():` block') self.app = WKitScope.app self.manager = WKitNetworkAccessManager(traffic_rules=traffic_rules) self.manager.finished.connect(self.handle_finished_network_reply) self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) self.page = WKitWebPage() self.page.setNetworkAccessManager(self.manager) self.page.loadFinished.connect(self.handle_page_load_finished) self.view = WKitWebView() self.view.setPage(self.page) self.view.setApplication(self.app) self._response = None self.gui = gui if gui: self.view.show()
def setAllCookies(self, cookieList): if not self.__read_only: QNetworkCookieJar.setAllCookies(self, cookieList) self.emit(SIGNAL('cookieJarUpdated()'))
def allCookies(self): return QNetworkCookieJar.allCookies(self)
class Ghost(object): """Ghost manages a QWebPage. :param user_agent: The default User-Agent header. :param wait_timeout: Maximum step duration in second. :param wait_callback: An optional callable that is periodically executed until Ghost stops waiting. :param log_level: The optional logging level. :param display: A boolean that tells ghost to displays UI. :param viewport_size: A tuple that sets initial viewport size. :param ignore_ssl_errors: A boolean that forces ignore ssl errors. :param cache_dir: A directory path where to store cache datas. :param plugins_enabled: Enable plugins (like Flash). :param java_enabled: Enable Java JRE. :param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins']) :param download_images: Indicate if the browser should download images """ _alert = None _confirm_expected = None _prompt_expected = None _upload_file = None _app = None def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), plugins_enabled=False, java_enabled=False, plugin_path=[ '/usr/lib/mozilla/plugins', ], download_images=True, qt_debug=False): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Error('Xvfb is required to a ghost run outside ' + 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) if plugin_path: for p in plugin_path: Ghost._app.addLibraryPath(p) self.popup_messages = [] self.page = GhostWebPage(Ghost._app, self) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute( QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) self.page.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, download_images) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: class MyQWebView(QtWebKit.QWebView): def sizeHint(self): return QSize(*viewport_size) self.webview = MyQWebView() if plugins_enabled: self.webview.settings().setAttribute( QtWebKit.QWebSettings.PluginsEnabled, True) if java_enabled: self.webview.settings().setAttribute( QtWebKit.QWebSettings.JavaEnabled, True) self.webview.setPage(self.page) self.webview.show() else: self.webview = None def __del__(self): self.exit() def capture(self, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Returns snapshot as QImage. :param region: An optional tuple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ if region is None and selector is not None: region = self.region_for_selector(selector) if region: x1, y1, x2, y2 = region w, h = (x2 - x1), (y2 - y1) image = QImage(QSize(x2, y2), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() image = image.copy(x1, y1, w, h) else: self.main_frame.setScrollBarPolicy(QtCore.Qt.Vertical, QtCore.Qt.ScrollBarAlwaysOff) self.main_frame.setScrollBarPolicy(QtCore.Qt.Horizontal, QtCore.Qt.ScrollBarAlwaysOff) self.page.setViewportSize(self.main_frame.contentsSize()) image = QImage(self.page.viewportSize(), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() return image def capture_to(self, path, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Saves snapshot as image. :param path: The destination path. :param region: An optional tuple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ self.capture(region=region, format=format, selector=selector).save(path) def print_to_pdf( self, path, paper_size=(8.5, 11.0), paper_margins=(0, 0, 0, 0), paper_units=QPrinter.Inch, zoom_factor=1.0, ): """Saves page as a pdf file. See qt4 QPrinter documentation for more detailed explanations of options. :param path: The destination path. :param paper_size: A 2-tuple indicating size of page to print to. :param paper_margins: A 4-tuple indicating size of each margin. :param paper_units: Units for pager_size, pager_margins. :param zoom_factor: Scale the output content. """ assert len(paper_size) == 2 assert len(paper_margins) == 4 printer = QPrinter(mode=QPrinter.ScreenResolution) printer.setOutputFormat(QPrinter.PdfFormat) printer.setPaperSize(QtCore.QSizeF(*paper_size), paper_units) printer.setPageMargins(*(paper_margins + (paper_units, ))) printer.setFullPage(True) printer.setOutputFileName(path) if self.webview is None: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.setZoomFactor(zoom_factor) self.webview.print_(printer) @can_load_page def click(self, selector): """Click the targeted element. :param selector: A CSS3 selector to targeted element. """ if not self.exists(selector): raise Error("Can't find element to click") return self.evaluate(""" var element = document.querySelector("%s"); var evt = document.createEvent("MouseEvents"); evt.initMouseEvent("click", true, true, window, 1, 1, 1, 1, 1, false, false, false, false, 0, element); element.dispatchEvent(evt) """ % selector) class confirm: """Statement that tells Ghost how to deal with javascript confirm(). :param confirm: A boolean to set confirmation. :param callable: A callable that returns a boolean for confirmation. """ def __init__(self, confirm=True, callback=None): self.confirm = confirm self.callback = callback def __enter__(self): Ghost._confirm_expected = (self.confirm, self.callback) def __exit__(self, type, value, traceback): Ghost._confirm_expected = None @property def content(self, to_unicode=True): """Returns current frame HTML as a string. :param to_unicode: Whether to convert html to unicode or not """ if to_unicode: return unicode(self.main_frame.toHtml()) else: return self.main_frame.toHtml() @property def cookies(self): """Returns all cookies.""" return self.cookie_jar.allCookies() def delete_cookies(self): """Deletes all cookies.""" self.cookie_jar.setAllCookies([]) def clear_alert_message(self): """Clears the alert message""" self._alert = None @can_load_page def evaluate(self, script): """Evaluates script in page frame. :param script: The script to evaluate. """ return (self.main_frame.evaluateJavaScript("%s" % script), self._release_last_resources()) def evaluate_js_file(self, path, encoding='utf-8'): """Evaluates javascript file at given path in current frame. Raises native IOException in case of invalid file. :param path: The path of the file. :param encoding: The file's encoding. """ self.evaluate(codecs.open(path, encoding=encoding).read()) def exists(self, selector): """Checks if element exists for given selector. :param string: The element selector. """ return not self.main_frame.findFirstElement(selector).isNull() def exit(self): """Exits application and related.""" if self.display: self.webview.close() Ghost._app.quit() del self.manager del self.page del self.main_frame if hasattr(self, 'xvfb'): self.xvfb.terminate() @can_load_page def fill(self, selector, values): """Fills a form with provided values. :param selector: A CSS selector to the target form to fill. :param values: A dict containing the values. """ if not self.exists(selector): raise Error("Can't find form") resources = [] for field in values: r, res = self.set_field_value("%s [name=%s]" % (selector, field), values[field]) resources.extend(res) return True, resources @can_load_page def fire_on(self, selector, method): """Call method on element matching given selector. :param selector: A CSS selector to the target element. :param method: The name of the method to fire. :param expect_loading: Specifies if a page loading is expected. """ return self.evaluate('document.querySelector("%s").%s();' % \ (selector, method)) def global_exists(self, global_name): """Checks if javascript global exists. :param global_name: The name of the global. """ return self.evaluate('!(typeof %s === "undefined");' % global_name)[0] def hide(self): """Close the webview.""" try: self.webview.close() except: raise Error("no webview to close") def load_cookies(self, cookie_storage, keep_old=False): """load from cookielib's CookieJar or Set-Cookie3 format text file. :param cookie_storage: file location string on disk or CookieJar instance. :param keep_old: Don't reset, keep cookies not overridden. """ def toQtCookieJar(PyCookieJar, QtCookieJar): allCookies = QtCookieJar.cookies if keep_old else [] for pc in PyCookieJar: qc = toQtCookie(pc) allCookies.append(qc) QtCookieJar.setAllCookies(allCookies) def toQtCookie(PyCookie): qc = QNetworkCookie(PyCookie.name, PyCookie.value) qc.setSecure(PyCookie.secure) if PyCookie.path_specified: qc.setPath(PyCookie.path) if PyCookie.domain != "": qc.setDomain(PyCookie.domain) if PyCookie.expires != 0: t = QDateTime() t.setTime_t(PyCookie.expires) qc.setExpirationDate(t) # not yet handled(maybe less useful): # py cookie.rest / QNetworkCookie.setHttpOnly() return qc if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) cj.load() toQtCookieJar(cj, self.cookie_jar) elif cookie_storage.__class__.__name__.endswith('CookieJar'): toQtCookieJar(cookie_storage, self.cookie_jar) else: raise ValueError, 'unsupported cookie_storage type.' def open(self, address, method='get', headers={}, auth=None, body=None, default_popup_response=None): """Opens a web page. :param address: The resource URL. :param method: The Http method. :param headers: An optional dict of extra request hearders. :param auth: An optional tuple of HTTP auth (username, password). :param body: An optional string containing a payload. :param default_popup_response: the default response for any confirm/ alert/prompt popup from the Javascript (replaces the need for the with blocks) :return: Page resource, All loaded resources. """ body = body or QByteArray() try: method = getattr(QNetworkAccessManager, "%sOperation" % method.capitalize()) except AttributeError: raise Error("Invalid http method %s" % method) request = QNetworkRequest(QUrl(address)) request.CacheLoadControl(0) for header in headers: request.setRawHeader(header, headers[header]) self._auth = auth self._auth_attempt = 0 # Avoids reccursion self.main_frame.load(request, method, body) self.loaded = False if default_popup_response is not None: Ghost._prompt_expected = (default_popup_response, None) Ghost._confirm_expected = (default_popup_response, None) return self.wait_for_page_loaded() class prompt: """Statement that tells Ghost how to deal with javascript prompt(). :param value: A string value to fill in prompt. :param callback: A callable that returns the value to fill in. """ def __init__(self, value='', callback=None): self.value = value self.callback = callback def __enter__(self): Ghost._prompt_expected = (self.value, self.callback) def __exit__(self, type, value, traceback): Ghost._prompt_expected = None def region_for_selector(self, selector): """Returns frame region for given selector as tuple. :param selector: The targeted element. """ geo = self.main_frame.findFirstElement(selector).geometry() try: region = (geo.left(), geo.top(), geo.right(), geo.bottom()) except: raise Error("can't get region for selector '%s'" % selector) return region def save_cookies(self, cookie_storage): """Save to cookielib's CookieJar or Set-Cookie3 format text file. :param cookie_storage: file location string or CookieJar instance. """ def toPyCookieJar(QtCookieJar, PyCookieJar): for c in QtCookieJar.allCookies(): PyCookieJar.set_cookie(toPyCookie(c)) def toPyCookie(QtCookie): port = None port_specified = False secure = QtCookie.isSecure() name = str(QtCookie.name()) value = str(QtCookie.value()) v = str(QtCookie.path()) path_specified = bool(v != "") path = v if path_specified else None v = str(QtCookie.domain()) domain_specified = bool(v != "") domain = v domain_initial_dot = v.startswith( '.') if domain_specified else None v = long(QtCookie.expirationDate().toTime_t()) # Long type boundary on 32bit platfroms; avoid ValueError expires = 2147483647 if v > 2147483647 else v rest = {} discard = False return Cookie(0, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, None, None, rest) if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) toPyCookieJar(self.cookie_jar, cj) cj.save() elif cookie_storage.__class__.__name__.endswith('CookieJar'): toPyCookieJar(self.cookie_jar, cookie_storage) else: raise ValueError, 'unsupported cookie_storage type.' @can_load_page def set_field_value(self, selector, value, blur=True): """Sets the value of the field matched by given selector. :param selector: A CSS selector that target the field. :param value: The value to fill in. :param blur: An optional boolean that force blur when filled in. """ def _set_checkbox_value(el, value): el.setFocus() if value is True: el.setAttribute('checked', 'checked') else: el.removeAttribute('checked') def _set_checkboxes_value(els, value): for el in els: if el.attribute('value') == value: _set_checkbox_value(el, True) else: _set_checkbox_value(el, False) def _set_radio_value(els, value): for el in els: if el.attribute('value') == value: el.setFocus() el.setAttribute('checked', 'checked') def _set_text_value(el, value): el.setFocus() el.setAttribute('value', value) def _set_select_value(el, value): el.setFocus() self.evaluate( 'document.querySelector("%s").value = "%s";' % (selector.replace('"', '\"'), value.replace('"', '\"'))) def _set_textarea_value(el, value): el.setFocus() el.setPlainText(value) res, ressources = None, [] element = self.main_frame.findFirstElement(selector) if element.isNull(): raise Error('can\'t find element for %s"' % selector) if element.tagName() == "SELECT": _set_select_value(element, value) elif element.tagName() == "TEXTAREA": _set_textarea_value(element, value) elif element.tagName() == "INPUT": if element.attribute('type') in [ "color", "date", "datetime", "datetime-local", "email", "hidden", "month", "number", "password", "range", "search", "tel", "text", "time", "url", "week" ]: _set_text_value(element, value) elif element.attribute('type') == "checkbox": els = self.main_frame.findAllElements(selector) if els.count() > 1: _set_checkboxes_value(els, value) else: _set_checkbox_value(element, value) elif element.attribute('type') == "radio": _set_radio_value(self.main_frame.findAllElements(selector), value) elif element.attribute('type') == "file": Ghost._upload_file = value res, resources = self.click(selector) Ghost._upload_file = None else: raise Error('unsuported field tag') if blur: self.fire_on(selector, 'blur') return res, ressources def set_proxy(self, type, host='localhost', port=8888, user='', password=''): """Set up proxy for FURTHER connections. :param type: proxy type to use: \ none/default/socks5/https/http. :param host: proxy server ip or host name. :param port: proxy port. """ _types = { 'default': QNetworkProxy.DefaultProxy, 'none': QNetworkProxy.NoProxy, 'socks5': QNetworkProxy.Socks5Proxy, 'https': QNetworkProxy.HttpProxy, 'http': QNetworkProxy.HttpCachingProxy } if type is None: type = 'none' type = type.lower() if type in ['none', 'default']: self.manager.setProxy(QNetworkProxy(_types[type])) return elif type in _types: proxy = QNetworkProxy(_types[type], hostName=host, port=port, user=user, password=password) self.manager.setProxy(proxy) else: raise ValueError, 'Unsupported proxy type:' + type \ + '\nsupported types are: none/socks5/http/https/default' def set_viewport_size(self, width, height): """Sets the page viewport size. :param width: An integer that sets width pixel count. :param height: An integer that sets height pixel count. """ self.page.setViewportSize(QSize(width, height)) def append_popup_message(self, message): self.popup_messages.append(unicode(message)) def show(self): """Show current page inside a QWebView. """ self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() def sleep(self, value): started_at = time.time() while True: if time.time() > (started_at + value): break time.sleep(0.01) Ghost._app.processEvents() def wait_for(self, condition, timeout_message): """Waits until condition is True. :param condition: A callable that returns the condition. :param timeout_message: The exception message on timeout. """ started_at = time.time() while not condition(): if time.time() > (started_at + self.wait_timeout): raise TimeoutError(timeout_message) time.sleep(0.01) Ghost._app.processEvents() if self.wait_callback is not None: self.wait_callback() def wait_for_alert(self): """Waits for main frame alert(). """ self.wait_for(lambda: Ghost._alert is not None, 'User has not been alerted.') msg = Ghost._alert Ghost._alert = None return msg, self._release_last_resources() def wait_for_page_loaded(self): """Waits until page is loaded, assumed that a page as been requested. """ self.wait_for(lambda: self.loaded, 'Unable to load requested page') resources = self._release_last_resources() page = None url = self.main_frame.url().toString() url_without_hash = url.split("#")[0] for resource in resources: if url == resource.url or url_without_hash == resource.url: page = resource return page, resources def wait_for_selector(self, selector): """Waits until selector match an element on the frame. :param selector: The selector to wait for. """ self.wait_for(lambda: self.exists(selector), 'Can\'t find element matching "%s"' % selector) return True, self._release_last_resources() def wait_for_text(self, text): """Waits until given text appear on main frame. :param text: The text to wait for. """ self.wait_for(lambda: text in self.content, 'Can\'t find "%s" in current frame' % text) return True, self._release_last_resources() def _authenticate(self, mix, authenticator): """Called back on basic / proxy http auth. :param mix: The QNetworkReply or QNetworkProxy object. :param authenticator: The QAuthenticator object. """ if self._auth_attempt == 0: username, password = self._auth authenticator.setUser(username) authenticator.setPassword(password) self._auth_attempt += 1 def _page_loaded(self): """Called back when page is loaded. """ self.loaded = True self.cache.clear() def _page_load_started(self): """Called back when page load started. """ self.loaded = False def _release_last_resources(self): """Releases last loaded resources. :return: The released resources. """ last_resources = self.http_resources self.http_resources = [] return last_resources def _request_ended(self, reply): """Adds an HttpResource object to http_resources. :param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): Logger.log("[%s] bytesAvailable()= %s" % (str(reply.url()), reply.bytesAvailable()), level="debug") # Some web pages return cache headers that mandates not to cache the # reply, which means we won't find this QNetworkReply in the cache # object. In this case bytesAvailable will return > 0. # Such pages are www.etsy.com # This is a bit of a hack and due to the async nature of QT, might # not work at times. We should move to using some proxied implementation # of QNetworkManager and QNetworkReply in order to get the contents # of the requests properly rather than relying on the cache. if reply.bytesAvailable() > 0: content = reply.peek(reply.bytesAvailable()) else: content = None self.http_resources.append( HttpResource(reply, self.cache, content=content)) def _unsupported_content(self, reply): """Adds an HttpResource object to http_resources with unsupported content. :param reply: The QNetworkReply object. """ self.wait_for(lambda: reply.isFinished(), 'Download timeout.') if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append( HttpResource(reply, self.cache, reply.readAll())) def _on_manager_ssl_errors(self, reply, errors): url = unicode(reply.url().toString()) if self.ignore_ssl_errors: reply.ignoreSslErrors() else: Logger.log('SSL certificate error: %s' % url, level='warning')
class Ghost(object): """Ghost manages a QWebPage. :param user_agent: The default User-Agent header. :param wait_timeout: Maximum step duration in second. :param wait_callback: An optional callable that is periodically executed until Ghost stops waiting. :param log_level: The optional logging level. :param display: A boolean that tells ghost to displays UI. :param viewport_size: A tupple that sets initial viewport size. """ _alert = None _confirm_expected = None _prompt_expected = None _upload_file = None _app = None def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800,600)): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) self.page = GhostWebPage(Ghost._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0); self.set_viewport_size(*viewport_size) self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() def __del__(self): self.exit() def capture(self, region=None, selector=None, format=QImage.Format_ARGB32): """Returns snapshot as QImage. :param region: An optional tupple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ if region is None and selector is not None: region = self.region_for_selector(selector) if region: x1, y1, x2, y2 = region w, h = (x2 - x1), (y2 - y1) image = QImage(QSize(x2, y2), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() image = image.copy(x1, y1, w, h) else: image = QImage(self.page.viewportSize(), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() return image def capture_to(self, path, region=None, selector=None, format=QImage.Format_ARGB32): """Saves snapshot as image. :param path: The destination path. :param region: An optional tupple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ self.capture(region=region, format=format, selector=selector).save(path) @client_utils_required @can_load_page def click(self, selector): """Click the targeted element. :param selector: A CSS3 selector to targeted element. """ if not self.exists(selector): raise Exception("Can't find element to click") return self.evaluate('GhostUtils.click("%s");' % selector) class confirm: """Statement that tells Ghost how to deal with javascript confirm(). :param confirm: A bollean that confirm. :param callable: A callable that returns a boolean for confirmation. """ def __init__(self, confirm=True, callback=None): self.confirm = confirm self.callback = callback def __enter__(self): Ghost._confirm_expected = (self.confirm, self.callback) def __exit__(self, type, value, traceback): Ghost._confirm_expected = None @property def content(self): """Returns current frame HTML as a string.""" return unicode(self.main_frame.toHtml()) @property def cookies(self): """Returns all cookies.""" return self.cookie_jar.allCookies() def delete_cookies(self): """Deletes all cookies.""" self.cookie_jar.setAllCookies([]) @can_load_page def evaluate(self, script): """Evaluates script in page frame. :param script: The script to evaluate. """ return (self.main_frame.evaluateJavaScript("%s" % script), self._release_last_resources()) def evaluate_js_file(self, path, encoding='utf-8'): """Evaluates javascript file at given path in current frame. Raises native IOException in case of invalid file. :param path: The path of the file. :param encoding: The file's encoding. """ self.evaluate(codecs.open(path, encoding=encoding).read()) def exists(self, selector): """Checks if element exists for given selector. :param string: The element selector. """ return not self.main_frame.findFirstElement(selector).isNull() def exit(self): """Exits application and relateds.""" if self.display: self.webview.close() Ghost._app.exit() del self.manager del self.page del self.main_frame if hasattr(self, 'xvfb'): self.xvfb.terminate() @can_load_page def fill(self, selector, values): """Fills a form with provided values. :param selector: A CSS selector to the target form to fill. :param values: A dict containing the values. """ if not self.exists(selector): raise Exception("Can't find form") resources = [] for field in values: r, res = self.set_field_value("%s [name=%s]" % (selector, field), values[field]) resources.extend(res) return True, resources @client_utils_required @can_load_page def fire_on(self, selector, method): """Call method on element matching given selector. :param selector: A CSS selector to the target element. :param method: The name of the method to fire. :param expect_loading: Specifies if a page loading is expected. """ return self.evaluate('GhostUtils.fireOn("%s", "%s");' % ( selector, method)) def global_exists(self, global_name): """Checks if javascript global exists. :param global_name: The name of the global. """ return self.evaluate('!(typeof %s === "undefined");' % global_name)[0] def hide(self): """Close the webview.""" try: self.webview.close() except: raise Exception("no webview to close") def open(self, address, method='get', headers={}, auth=None): """Opens a web page. :param address: The resource URL. :param method: The Http method. :param headers: An optional dict of extra request hearders. :param auth: An optional tupple of HTTP auth (Basic, username, password). :return: Page resource, All loaded resources. """ body = QByteArray() try: method = getattr(QNetworkAccessManager, "%sOperation" % method.capitalize()) except AttributeError: raise Exception("Invalid http method %s" % method) request = QNetworkRequest(QUrl(address)) if auth is not None: auth_type, username, password = auth def authenticate(reply, authenticator): authenticator.setUser(username) authenticator.setPassword(password) self.page.networkAccessManager().authenticationRequired.connect(authenticate) if not "User-Agent" in headers: headers["User-Agent"] = self.user_agent for header in headers: request.setRawHeader(header, headers[header]) self.main_frame.load(request, method, body) self.loaded = False return self.wait_for_page_loaded() class prompt: """Statement that tells Ghost how to deal with javascript prompt(). :param value: A string value to fill in prompt. :param callback: A callable that returns the value to fill in. """ def __init__(self, value='', callback=None): self.value = value self.callback = callback def __enter__(self): Ghost._prompt_expected = (self.value, self.callback) def __exit__(self, type, value, traceback): Ghost._prompt_expected = None @client_utils_required def region_for_selector(self, selector): """Returns frame region for given selector as tupple. :param selector: The targeted element. """ geo = self.main_frame.findFirstElement(selector).geometry() try: region = (geo.left(), geo.top(), geo.right(), geo.bottom()) except: raise Exception("can't get region for selector '%s'" % selector) return region @can_load_page @client_utils_required def set_field_value(self, selector, value, blur=True): """Sets the value of the field matched by given selector. :param selector: A CSS selector that target the field. :param value: The value to fill in. :param blur: An optional boolean that force blur when filled in. """ def _set_text_value(selector, value): return self.evaluate( 'document.querySelector("%s").value=%s;' % (selector, json.dumps(value))) res, resources = None, [] element = self.main_frame.findFirstElement(selector) if element.isNull(): raise Exception('can\'t find element for %s"' % selector) self.fire_on(selector, 'focus') if element.tagName() in ["TEXTAREA", "SELECT"]: res, resources = _set_text_value(selector, value) elif element.tagName() == "INPUT": if element.attribute('type') in ["color", "date", "datetime", "datetime-local", "email", "hidden", "month", "number", "password", "range", "search", "tel", "text", "time", "url", "week"]: res, resources = _set_text_value(selector, value) elif element.attribute('type') == "checkbox": res, resources = self.evaluate( 'GhostUtils.setCheckboxValue("%s", %s);' % (selector, json.dumps(value))) elif element.attribute('type') == "radio": res, resources = self.evaluate( 'GhostUtils.setRadioValue("%s", %s);' % (selector, json.dumps(value))) elif element.attribute('type') == "file": Ghost._upload_file = value res, resources = self.click(selector) Ghost._upload_file = None else: raise Exception('unsuported field tag') if blur: self.fire_on(selector, 'blur') return res, resources def set_viewport_size(self, width, height): """Sets the page viewport size. :param width: An integer that sets width pixel count. :param height: An integer that sets height pixel count. """ self.page.setViewportSize(QSize(width, height)) def show(self): """Show current page inside a QWebView. """ self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() def wait_for_alert(self): """Waits for main frame alert(). """ self._wait_for(lambda: Ghost._alert is not None, 'User has not been alerted.') msg = Ghost._alert Ghost._alert = None return msg, self._release_last_resources() def wait_for_page_loaded(self): """Waits until page is loaded, assumed that a page as been requested. """ self._wait_for(lambda: self.loaded, 'Unable to load requested page') resources = self._release_last_resources() page = None for resource in resources: if self.main_frame.url().toString() == resource.url: page = resource return page, resources def wait_for_selector(self, selector): """Waits until selector match an element on the frame. :param selector: The selector to wait for. """ self._wait_for(lambda: self.exists(selector), 'Can\'t find element matching "%s"' % selector) return True, self._release_last_resources() def wait_for_text(self, text): """Waits until given text appear on main frame. :param text: The text to wait for. """ self._wait_for(lambda: text in self.content, 'Can\'t find "%s" in current frame' % text) return True, self._release_last_resources() def _release_last_resources(self): """Releases last loaded resources. :return: The released resources. """ last_resources = self.http_resources self.http_resources = [] return last_resources def _page_loaded(self): """Called back when page is loaded. """ self.loaded = True def _page_load_started(self): """Called back when page load started. """ self.loaded = False def _request_ended(self, res): """Adds an HttpResource object to http_resources. :param res: The request result. """ if res.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(res)) def _wait_for(self, condition, timeout_message): """Waits until condition is True. :param condition: A callable that returns the condition. :param timeout_message: The exception message on timeout. """ started_at = time.time() while not condition(): if time.time() > (started_at + self.wait_timeout): raise Exception(timeout_message) time.sleep(0.01) Ghost._app.processEvents() if self.wait_callback is not None: self.wait_callback()
class Ghost(object): """Ghost manages a QWebPage. :param user_agent: The default User-Agent header. :param wait_timeout: Maximum step duration in second. :param wait_callback: An optional callable that is periodically executed until Ghost stops waiting. :param log_level: The optional logging level. :param display: A boolean that tells ghost to displays UI. :param viewport_size: A tuple that sets initial viewport size. :param ignore_ssl_errors: A boolean that forces ignore ssl errors. :param cache_dir: A directory path where to store cache datas. :param plugins_enabled: Enable plugins (like Flash). :param java_enabled: Enable Java JRE. :param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins']) :param download_images: Indicate if the browser should download images """ _alert = None _confirm_expected = None _prompt_expected = None _upload_file = None _app = None def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), plugins_enabled=False, java_enabled=False, plugin_path=['/usr/lib/mozilla/plugins',], download_images=True, qt_debug=False): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Error('Xvfb is required to a ghost run outside ' + 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) if plugin_path: for p in plugin_path: Ghost._app.addLibraryPath(p) self.popup_messages = [] self.page = GhostWebPage(Ghost._app, self) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) self.page.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, download_images) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: class MyQWebView(QtWebKit.QWebView): def sizeHint(self): return QSize(*viewport_size) self.webview = MyQWebView() if plugins_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True) if java_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, True) self.webview.setPage(self.page) self.webview.show() else: self.webview = None def __del__(self): self.exit() def capture(self, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Returns snapshot as QImage. :param region: An optional tuple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ if region is None and selector is not None: region = self.region_for_selector(selector) if region: x1, y1, x2, y2 = region w, h = (x2 - x1), (y2 - y1) image = QImage(QSize(x2, y2), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() image = image.copy(x1, y1, w, h) else: self.main_frame.setScrollBarPolicy(QtCore.Qt.Vertical, QtCore.Qt.ScrollBarAlwaysOff) self.main_frame.setScrollBarPolicy(QtCore.Qt.Horizontal, QtCore.Qt.ScrollBarAlwaysOff) self.page.setViewportSize(self.main_frame.contentsSize()) image = QImage(self.page.viewportSize(), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() return image def capture_to(self, path, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Saves snapshot as image. :param path: The destination path. :param region: An optional tuple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ self.capture(region=region, format=format, selector=selector).save(path) def print_to_pdf(self, path, paper_size = (8.5, 11.0), paper_margins = (0, 0, 0, 0), paper_units = QPrinter.Inch, zoom_factor = 1.0, ): """Saves page as a pdf file. See qt4 QPrinter documentation for more detailed explanations of options. :param path: The destination path. :param paper_size: A 2-tuple indicating size of page to print to. :param paper_margins: A 4-tuple indicating size of each margin. :param paper_units: Units for pager_size, pager_margins. :param zoom_factor: Scale the output content. """ assert len(paper_size) == 2 assert len(paper_margins) == 4 printer = QPrinter(mode = QPrinter.ScreenResolution) printer.setOutputFormat(QPrinter.PdfFormat) printer.setPaperSize(QtCore.QSizeF(*paper_size), paper_units) printer.setPageMargins(*(paper_margins + (paper_units,))) printer.setFullPage(True) printer.setOutputFileName(path) if self.webview is None: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.setZoomFactor(zoom_factor) self.webview.print_(printer) @can_load_page def click(self, selector): """Click the targeted element. :param selector: A CSS3 selector to targeted element. """ if not self.exists(selector): raise Error("Can't find element to click") return self.evaluate(""" var element = document.querySelector("%s"); var evt = document.createEvent("MouseEvents"); evt.initMouseEvent("click", true, true, window, 1, 1, 1, 1, 1, false, false, false, false, 0, element); element.dispatchEvent(evt) """ % selector) class confirm: """Statement that tells Ghost how to deal with javascript confirm(). :param confirm: A boolean to set confirmation. :param callable: A callable that returns a boolean for confirmation. """ def __init__(self, confirm=True, callback=None): self.confirm = confirm self.callback = callback def __enter__(self): Ghost._confirm_expected = (self.confirm, self.callback) def __exit__(self, type, value, traceback): Ghost._confirm_expected = None @property def content(self, to_unicode=True): """Returns current frame HTML as a string. :param to_unicode: Whether to convert html to unicode or not """ if to_unicode: return unicode(self.main_frame.toHtml()) else: return self.main_frame.toHtml() @property def cookies(self): """Returns all cookies.""" return self.cookie_jar.allCookies() def delete_cookies(self): """Deletes all cookies.""" self.cookie_jar.setAllCookies([]) def clear_alert_message(self): """Clears the alert message""" self._alert = None @can_load_page def evaluate(self, script): """Evaluates script in page frame. :param script: The script to evaluate. """ return (self.main_frame.evaluateJavaScript("%s" % script), self._release_last_resources()) def evaluate_js_file(self, path, encoding='utf-8'): """Evaluates javascript file at given path in current frame. Raises native IOException in case of invalid file. :param path: The path of the file. :param encoding: The file's encoding. """ self.evaluate(codecs.open(path, encoding=encoding).read()) def exists(self, selector): """Checks if element exists for given selector. :param string: The element selector. """ return not self.main_frame.findFirstElement(selector).isNull() def exit(self): """Exits application and related.""" if self.display: self.webview.close() Ghost._app.quit() del self.manager del self.page del self.main_frame if hasattr(self, 'xvfb'): self.xvfb.terminate() @can_load_page def fill(self, selector, values): """Fills a form with provided values. :param selector: A CSS selector to the target form to fill. :param values: A dict containing the values. """ if not self.exists(selector): raise Error("Can't find form") resources = [] for field in values: r, res = self.set_field_value("%s [name=%s]" % (selector, field), values[field]) resources.extend(res) return True, resources @can_load_page def fire_on(self, selector, method): """Call method on element matching given selector. :param selector: A CSS selector to the target element. :param method: The name of the method to fire. :param expect_loading: Specifies if a page loading is expected. """ return self.evaluate('document.querySelector("%s").%s();' % \ (selector, method)) def global_exists(self, global_name): """Checks if javascript global exists. :param global_name: The name of the global. """ return self.evaluate('!(typeof %s === "undefined");' % global_name)[0] def hide(self): """Close the webview.""" try: self.webview.close() except: raise Error("no webview to close") def load_cookies( self, cookie_storage, keep_old=False ): """load from cookielib's CookieJar or Set-Cookie3 format text file. :param cookie_storage: file location string on disk or CookieJar instance. :param keep_old: Don't reset, keep cookies not overridden. """ def toQtCookieJar( PyCookieJar, QtCookieJar ): allCookies = QtCookieJar.cookies if keep_old else [] for pc in PyCookieJar: qc = toQtCookie(pc) allCookies.append(qc) QtCookieJar.setAllCookies(allCookies) def toQtCookie(PyCookie): qc = QNetworkCookie( PyCookie.name, PyCookie.value ) qc.setSecure(PyCookie.secure) if PyCookie.path_specified: qc.setPath(PyCookie.path) if PyCookie.domain != "" : qc.setDomain(PyCookie.domain) if PyCookie.expires != 0: t = QDateTime() t.setTime_t(PyCookie.expires) qc.setExpirationDate(t) # not yet handled(maybe less useful): # py cookie.rest / QNetworkCookie.setHttpOnly() return qc if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) cj.load() toQtCookieJar(cj, self.cookie_jar) elif cookie_storage.__class__.__name__.endswith('CookieJar') : toQtCookieJar(cookie_storage, self.cookie_jar) else: raise ValueError, 'unsupported cookie_storage type.' def open(self, address, method='get', headers={}, auth=None, body=None, default_popup_response=None): """Opens a web page. :param address: The resource URL. :param method: The Http method. :param headers: An optional dict of extra request hearders. :param auth: An optional tuple of HTTP auth (username, password). :param body: An optional string containing a payload. :param default_popup_response: the default response for any confirm/ alert/prompt popup from the Javascript (replaces the need for the with blocks) :return: Page resource, All loaded resources. """ body = body or QByteArray() try: method = getattr(QNetworkAccessManager, "%sOperation" % method.capitalize()) except AttributeError: raise Error("Invalid http method %s" % method) request = QNetworkRequest(QUrl(address)) request.CacheLoadControl(0) for header in headers: request.setRawHeader(header, headers[header]) self._auth = auth self._auth_attempt = 0 # Avoids reccursion self.main_frame.load(request, method, body) self.loaded = False Ghost._prompt_expected = (default_popup_response, None) Ghost._confirm_expected = (default_popup_response, None) return self.wait_for_page_loaded() class prompt: """Statement that tells Ghost how to deal with javascript prompt(). :param value: A string value to fill in prompt. :param callback: A callable that returns the value to fill in. """ def __init__(self, value='', callback=None): self.value = value self.callback = callback def __enter__(self): Ghost._prompt_expected = (self.value, self.callback) def __exit__(self, type, value, traceback): Ghost._prompt_expected = None def region_for_selector(self, selector): """Returns frame region for given selector as tuple. :param selector: The targeted element. """ geo = self.main_frame.findFirstElement(selector).geometry() try: region = (geo.left(), geo.top(), geo.right(), geo.bottom()) except: raise Error("can't get region for selector '%s'" % selector) return region def save_cookies(self, cookie_storage): """Save to cookielib's CookieJar or Set-Cookie3 format text file. :param cookie_storage: file location string or CookieJar instance. """ def toPyCookieJar(QtCookieJar, PyCookieJar): for c in QtCookieJar.allCookies(): PyCookieJar.set_cookie(toPyCookie(c)) def toPyCookie(QtCookie): port=None port_specified=False secure=QtCookie.isSecure() name=str(QtCookie.name()) value=str(QtCookie.value()) v = str(QtCookie.path()) path_specified = bool( v != "" ) path = v if path_specified else None v = str(QtCookie.domain()) domain_specified = bool( v != "" ) domain = v domain_initial_dot = v.startswith('.') if domain_specified else None v = long(QtCookie.expirationDate().toTime_t()) # Long type boundary on 32bit platfroms; avoid ValueError expires = 2147483647 if v > 2147483647 else v rest = {} discard = False return Cookie(0, name, value, port, port_specified, domain , domain_specified, domain_initial_dot, path, path_specified , secure, expires, discard, None, None, rest) if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) toPyCookieJar(self.cookie_jar,cj) cj.save() elif cookie_storage.__class__.__name__.endswith('CookieJar') : toPyCookieJar(self.cookie_jar,cookie_storage) else: raise ValueError, 'unsupported cookie_storage type.' @can_load_page def set_field_value(self, selector, value, blur=True): """Sets the value of the field matched by given selector. :param selector: A CSS selector that target the field. :param value: The value to fill in. :param blur: An optional boolean that force blur when filled in. """ def _set_checkbox_value(el, value): el.setFocus() if value is True: el.setAttribute('checked', 'checked') else: el.removeAttribute('checked') def _set_checkboxes_value(els, value): for el in els: if el.attribute('value') == value: _set_checkbox_value(el, True) else: _set_checkbox_value(el, False) def _set_radio_value(els, value): for el in els: if el.attribute('value') == value: el.setFocus() el.setAttribute('checked', 'checked') def _set_text_value(el, value): el.setFocus() el.setAttribute('value', value) def _set_select_value(el, value): el.setFocus() self.evaluate('document.querySelector("%s").value = "%s";' % (selector.replace('"', '\"'), value.replace('"', '\"'))) def _set_textarea_value(el, value): el.setFocus() el.setPlainText(value) res, ressources = None, [] element = self.main_frame.findFirstElement(selector) if element.isNull(): raise Error('can\'t find element for %s"' % selector) if element.tagName() == "SELECT": _set_select_value(element, value) elif element.tagName() == "TEXTAREA": _set_textarea_value(element, value) elif element.tagName() == "INPUT": if element.attribute('type') in ["color", "date", "datetime", "datetime-local", "email", "hidden", "month", "number", "password", "range", "search", "tel", "text", "time", "url", "week"]: _set_text_value(element, value) elif element.attribute('type') == "checkbox": els = self.main_frame.findAllElements(selector) if els.count() > 1: _set_checkboxes_value(els, value) else: _set_checkbox_value(element, value) elif element.attribute('type') == "radio": _set_radio_value(self.main_frame.findAllElements(selector), value) elif element.attribute('type') == "file": Ghost._upload_file = value res, resources = self.click(selector) Ghost._upload_file = None else: raise Error('unsuported field tag') if blur: self.fire_on(selector, 'blur') return res, ressources def set_proxy(self, type, host='localhost', port=8888, user='', password=''): """Set up proxy for FURTHER connections. :param type: proxy type to use: \ none/default/socks5/https/http. :param host: proxy server ip or host name. :param port: proxy port. """ _types = {'default': QNetworkProxy.DefaultProxy, 'none': QNetworkProxy.NoProxy, 'socks5': QNetworkProxy.Socks5Proxy, 'https': QNetworkProxy.HttpProxy, 'http': QNetworkProxy.HttpCachingProxy } if type is None: type='none' type = type.lower() if type in ['none','default'] : self.manager.setProxy(QNetworkProxy(_types[type])) return elif type in _types: proxy = QNetworkProxy(_types[type], hostName=host, port=port , user=user, password=password ) self.manager.setProxy(proxy) else: raise ValueError, 'Unsupported proxy type:' + type \ + '\nsupported types are: none/socks5/http/https/default' def set_viewport_size(self, width, height): """Sets the page viewport size. :param width: An integer that sets width pixel count. :param height: An integer that sets height pixel count. """ self.page.setViewportSize(QSize(width, height)) def append_popup_message(self, message): self.popup_messages.append(str(message)) def show(self): """Show current page inside a QWebView. """ self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() def sleep(self, value): started_at = time.time() while True: if time.time() > (started_at + value): break time.sleep(0.01) Ghost._app.processEvents() def wait_for(self, condition, timeout_message): """Waits until condition is True. :param condition: A callable that returns the condition. :param timeout_message: The exception message on timeout. """ started_at = time.time() while not condition(): if time.time() > (started_at + self.wait_timeout): raise TimeoutError(timeout_message) time.sleep(0.01) Ghost._app.processEvents() if self.wait_callback is not None: self.wait_callback() def wait_for_alert(self): """Waits for main frame alert(). """ self.wait_for(lambda: Ghost._alert is not None, 'User has not been alerted.') msg = Ghost._alert Ghost._alert = None return msg, self._release_last_resources() def wait_for_page_loaded(self): """Waits until page is loaded, assumed that a page as been requested. """ self.wait_for(lambda: self.loaded, 'Unable to load requested page') resources = self._release_last_resources() page = None url = self.main_frame.url().toString() url_without_hash = url.split("#")[0] for resource in resources: if url == resource.url or url_without_hash == resource.url: page = resource return page, resources def wait_for_selector(self, selector): """Waits until selector match an element on the frame. :param selector: The selector to wait for. """ self.wait_for(lambda: self.exists(selector), 'Can\'t find element matching "%s"' % selector) return True, self._release_last_resources() def wait_for_text(self, text): """Waits until given text appear on main frame. :param text: The text to wait for. """ self.wait_for(lambda: text in self.content, 'Can\'t find "%s" in current frame' % text) return True, self._release_last_resources() def _authenticate(self, mix, authenticator): """Called back on basic / proxy http auth. :param mix: The QNetworkReply or QNetworkProxy object. :param authenticator: The QAuthenticator object. """ if self._auth_attempt == 0: username, password = self._auth authenticator.setUser(username) authenticator.setPassword(password) self._auth_attempt += 1 def _page_loaded(self): """Called back when page is loaded. """ self.loaded = True self.cache.clear() def _page_load_started(self): """Called back when page load started. """ self.loaded = False def _release_last_resources(self): """Releases last loaded resources. :return: The released resources. """ last_resources = self.http_resources self.http_resources = [] return last_resources def _request_ended(self, reply): """Adds an HttpResource object to http_resources. :param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): Logger.log("[%s] bytesAvailable()= %s" %(str(reply.url()), reply.bytesAvailable()), level="debug") # Some web pages return cache headers that mandates not to cache the # reply, which means we won't find this QNetworkReply in the cache # object. In this case bytesAvailable will return > 0. # Such pages are www.etsy.com # This is a bit of a hack and due to the async nature of QT, might # not work at times. We should move to using some proxied implementation # of QNetworkManager and QNetworkReply in order to get the contents # of the requests properly rather than relying on the cache. if reply.bytesAvailable() > 0: content = reply.peek(reply.bytesAvailable()) else: content = None self.http_resources.append(HttpResource(reply, self.cache, content=content)) def _unsupported_content(self, reply): """Adds an HttpResource object to http_resources with unsupported content. :param reply: The QNetworkReply object. """ self.wait_for(lambda: reply.isFinished(), 'Download timeout.') if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache, reply.readAll())) def _on_manager_ssl_errors(self, reply, errors): url = unicode(reply.url().toString()) if self.ignore_ssl_errors: reply.ignoreSslErrors() else: Logger.log('SSL certificate error: %s' % url, level='warning')
class ShoutCastForm(PluginBase.PluginBase): '''Grab Shoutcast streams and save them as "bookmarks" - and play them on the currently selected server. General shoutcast information is not preserved between runs. Also, the shoutcast server/API is pretty lame so timeouts actually occur quite frequently. ''' moduleName = '&Shoutcast' moduleIcon = "network-workgroup" def load(self): pass def event(self, event): if event.type() == QEvent.Paint: if not hasattr(self, 'webView'): self._load() self.event = super(ShoutCastForm, self).event return False def _load(self): self.cookie = QNetworkCookie('Settings', 'Player~others|Bandwidth~ALL|Codec~ALL') self.cookie.setDomain('.shoutcast.com') self.cookie.setExpirationDate(QDateTime()) self.cookie.setPath('/') self.webView = QWebView(self) self.webPage = self.webView.page() self.cookiejar = QNetworkCookieJar() self.cookiejar.setAllCookies([self.cookie]) self.webPage.networkAccessManager().setCookieJar(self.cookiejar) self.layout = QVBoxLayout(self) self.layout.addWidget(self.webView) self.webView.load(QUrl(HOMEURL)) self.webPage.setLinkDelegationPolicy(QWebPage.DelegateExternalLinks) self.connect(self.webPage, SIGNAL('linkClicked(const QUrl&)'), self._processLink) def _processLink(self, url): if url.host() == TUNEIN: self._playStation(url.toString()) else: self.webView.load(url) self.webView.show() def _playStation(self, url): data = self._retreivePLS(url) adrlist = self._parsePLS(data) self.mpdclient.send('command_list_ok_begin') try: for address in adrlist: self.mpdclient.send('add', (address, )) finally: self.mpdclient.send('command_list_end') def _retreivePLS(self, url): conn = httplib.HTTPConnection(TUNEIN) conn.request("GET", TUNEINFORMAT % url.split('=')[-1]) resp = conn.getresponse() if resp.status == 200: return resp.read().split('\n') else: raise httplib.HTTPException('Got bad status code.') def _parsePLS(self, data): adrlist = [] state = '' while data: line = data.pop(0) if state == '' and line == '[playlist]': state = 'playlist' elif state == 'playlist': if '=' in line: key, value = line.split('=', 1) if key.startswith('File'): adrlist.append(value) else: raise httplib.HTTPException( 'Encountered error during parsing of the playlist.') return adrlist
def __init__(self, cookieFile, parent=None): QNetworkCookieJar.__init__(self, parent) self.m_cookieFile = cookieFile
class Ghost(object): """Ghost manages a QWebPage. :param user_agent: The default User-Agent header. :param wait_timeout: Maximum step duration in second. :param wait_callback: An optional callable that is periodically executed until Ghost stops waiting. :param log_level: The optional logging level. :param display: A boolean that tells ghost to displays UI. :param viewport_size: A tupple that sets initial viewport size. :param ignore_ssl_errors: A boolean that forces ignore ssl errors. :param cache_dir: A directory path where to store cache datas. :param plugins_enabled: Enable plugins (like Flash). :param java_enabled: Enable Java JRE. :param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins']) :param download_images: Indicate if the browser should download images """ _alert = None _confirm_expected = None _prompt_expected = None _upload_file = None _app = None def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), plugins_enabled=False, java_enabled=False, plugin_path=['/usr/lib/mozilla/plugins',], download_images=True): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) if plugin_path: for p in plugin_path: Ghost._app.addLibraryPath(p) self.page = GhostWebPage(Ghost._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) self.page.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, download_images) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: self.webview = QtWebKit.QWebView() if plugins_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True) if java_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, True) self.webview.setPage(self.page) self.webview.show() def __del__(self): self.exit() def capture(self, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Returns snapshot as QImage. :param region: An optional tupple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ if region is None and selector is not None: region = self.region_for_selector(selector) if region: x1, y1, x2, y2 = region w, h = (x2 - x1), (y2 - y1) image = QImage(QSize(x2, y2), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() image = image.copy(x1, y1, w, h) else: image = QImage(self.page.viewportSize(), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() return image def capture_to(self, path, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Saves snapshot as image. :param path: The destination path. :param region: An optional tupple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ self.capture(region=region, format=format, selector=selector).save(path) @can_load_page def click(self, selector): """Click the targeted element. :param selector: A CSS3 selector to targeted element. """ if not self.exists(selector): raise Exception("Can't find element to click") return self.evaluate(""" var element = document.querySelector("%s"); var evt = document.createEvent("MouseEvents"); evt.initMouseEvent("click", true, true, window, 1, 1, 1, 1, 1, false, false, false, false, 0, element); element.dispatchEvent(evt) """ % selector) class confirm: """Statement that tells Ghost how to deal with javascript confirm(). :param confirm: A bollean that confirm. :param callable: A callable that returns a boolean for confirmation. """ def __init__(self, confirm=True, callback=None): self.confirm = confirm self.callback = callback def __enter__(self): Ghost._confirm_expected = (self.confirm, self.callback) def __exit__(self, type, value, traceback): Ghost._confirm_expected = None @property def content(self): """Returns current frame HTML as a string.""" return unicode(self.main_frame.toHtml()) @property def cookies(self): """Returns all cookies.""" return self.cookie_jar.allCookies() def delete_cookies(self): """Deletes all cookies.""" self.cookie_jar.setAllCookies([]) @can_load_page def evaluate(self, script): """Evaluates script in page frame. :param script: The script to evaluate. """ return (self.main_frame.evaluateJavaScript("%s" % script), self._release_last_resources()) def evaluate_js_file(self, path, encoding='utf-8'): """Evaluates javascript file at given path in current frame. Raises native IOException in case of invalid file. :param path: The path of the file. :param encoding: The file's encoding. """ self.evaluate(codecs.open(path, encoding=encoding).read()) def exists(self, selector): """Checks if element exists for given selector. :param string: The element selector. """ return not self.main_frame.findFirstElement(selector).isNull() def exit(self): """Exits application and relateds.""" if self.display: self.webview.close() Ghost._app.quit() del self.manager del self.page del self.main_frame if hasattr(self, 'xvfb'): self.xvfb.terminate() @can_load_page def fill(self, selector, values): """Fills a form with provided values. :param selector: A CSS selector to the target form to fill. :param values: A dict containing the values. """ if not self.exists(selector): raise Exception("Can't find form") resources = [] for field in values: r, res = self.set_field_value("%s [name=%s]" % (selector, field), values[field]) resources.extend(res) return True, resources @can_load_page def fire_on(self, selector, method): """Call method on element matching given selector. :param selector: A CSS selector to the target element. :param method: The name of the method to fire. :param expect_loading: Specifies if a page loading is expected. """ return self.evaluate('document.querySelector("%s").%s();' % \ (selector, method)) def global_exists(self, global_name): """Checks if javascript global exists. :param global_name: The name of the global. """ return self.evaluate('!(typeof %s === "undefined");' % global_name)[0] def hide(self): """Close the webview.""" try: self.webview.close() except: raise Exception("no webview to close") def open(self, address, method='get', headers={}, auth=None, body=None): """Opens a web page. :param address: The resource URL. :param method: The Http method. :param headers: An optional dict of extra request hearders. :param auth: An optional tupple of HTTP auth (username, password). :param body: An optional string containing a payload. :return: Page resource, All loaded resources. """ body = body or QByteArray() try: method = getattr(QNetworkAccessManager, "%sOperation" % method.capitalize()) except AttributeError: raise Exception("Invalid http method %s" % method) request = QNetworkRequest(QUrl(address)) request.CacheLoadControl(0) for header in headers: request.setRawHeader(header, headers[header]) self._auth = auth self._auth_attempt = 0 # Avoids reccursion self.main_frame.load(request, method, body) self.loaded = False return self.wait_for_page_loaded() class prompt: """Statement that tells Ghost how to deal with javascript prompt(). :param value: A string value to fill in prompt. :param callback: A callable that returns the value to fill in. """ def __init__(self, value='', callback=None): self.value = value self.callback = callback def __enter__(self): Ghost._prompt_expected = (self.value, self.callback) def __exit__(self, type, value, traceback): Ghost._prompt_expected = None def region_for_selector(self, selector): """Returns frame region for given selector as tupple. :param selector: The targeted element. """ geo = self.main_frame.findFirstElement(selector).geometry() try: region = (geo.left(), geo.top(), geo.right(), geo.bottom()) except: raise Exception("can't get region for selector '%s'" % selector) return region @can_load_page def set_field_value(self, selector, value, blur=True): """Sets the value of the field matched by given selector. :param selector: A CSS selector that target the field. :param value: The value to fill in. :param blur: An optional boolean that force blur when filled in. """ def _set_checkbox_value(el, value): el.setFocus() if value is True: el.setAttribute('checked', 'checked') else: el.removeAttribute('checked') def _set_checkboxes_value(els, value): for el in els: if el.attribute('value') == value: _set_checkbox_value(el, True) else: _set_checkbox_value(el, False) def _set_radio_value(els, value): for el in els: if el.attribute('value') == value: el.setFocus() el.setAttribute('checked', 'checked') def _set_text_value(el, value): el.setFocus() el.setAttribute('value', value) def _set_textarea_value(el, value): el.setFocus() el.setPlainText(value) res, ressources = None, [] element = self.main_frame.findFirstElement(selector) if element.isNull(): raise Exception('can\'t find element for %s"' % selector) if element.tagName() == "SELECT": _set_text_value(element, value) elif element.tagName() == "TEXTAREA": _set_textarea_value(element, value) elif element.tagName() == "INPUT": if element.attribute('type') in ["color", "date", "datetime", "datetime-local", "email", "hidden", "month", "number", "password", "range", "search", "tel", "text", "time", "url", "week"]: _set_text_value(element, value) elif element.attribute('type') == "checkbox": els = self.main_frame.findAllElements(selector) if els.count() > 1: _set_checkboxes_value(els, value) else: _set_checkbox_value(element, value) elif element.attribute('type') == "radio": _set_radio_value(self.main_frame.findAllElements(selector), value) elif element.attribute('type') == "file": Ghost._upload_file = value res, resources = self.click(selector) Ghost._upload_file = None else: raise Exception('unsuported field tag') if blur: self.fire_on(selector, 'blur') return res, ressources def set_viewport_size(self, width, height): """Sets the page viewport size. :param width: An integer that sets width pixel count. :param height: An integer that sets height pixel count. """ self.page.setViewportSize(QSize(width, height)) def show(self): """Show current page inside a QWebView. """ self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() def wait_for(self, condition, timeout_message): """Waits until condition is True. :param condition: A callable that returns the condition. :param timeout_message: The exception message on timeout. """ started_at = time.time() while not condition(): if time.time() > (started_at + self.wait_timeout): raise Exception(timeout_message) time.sleep(0.01) Ghost._app.processEvents() if self.wait_callback is not None: self.wait_callback() def wait_for_alert(self): """Waits for main frame alert(). """ self.wait_for(lambda: Ghost._alert is not None, 'User has not been alerted.') msg = Ghost._alert Ghost._alert = None return msg, self._release_last_resources() def wait_for_page_loaded(self): """Waits until page is loaded, assumed that a page as been requested. """ self.wait_for(lambda: self.loaded, 'Unable to load requested page') resources = self._release_last_resources() page = None url = self.main_frame.url().toString() for resource in resources: if url == resource.url: page = resource return page, resources def wait_for_selector(self, selector): """Waits until selector match an element on the frame. :param selector: The selector to wait for. """ self.wait_for(lambda: self.exists(selector), 'Can\'t find element matching "%s"' % selector) return True, self._release_last_resources() def wait_for_text(self, text): """Waits until given text appear on main frame. :param text: The text to wait for. """ self.wait_for(lambda: text in self.content, 'Can\'t find "%s" in current frame' % text) return True, self._release_last_resources() def _authenticate(self, mix, authenticator): """Called back on basic / proxy http auth. :param mix: The QNetworkReply or QNetworkProxy object. :param authenticator: The QAuthenticator object. """ if self._auth_attempt == 0: username, password = self._auth authenticator.setUser(username) authenticator.setPassword(password) self._auth_attempt += 1 def _page_loaded(self): """Called back when page is loaded. """ self.loaded = True self.cache.clear() def _page_load_started(self): """Called back when page load started. """ self.loaded = False def _release_last_resources(self): """Releases last loaded resources. :return: The released resources. """ last_resources = self.http_resources self.http_resources = [] return last_resources def _request_ended(self, reply): """Adds an HttpResource object to http_resources. :param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache)) def _unsupported_content(self, reply): """Adds an HttpResource object to http_resources with unsupported content. :param reply: The QNetworkReply object. """ self.wait_for(lambda: reply.isFinished(), 'Download timeout.') if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache, reply.readAll())) def _on_manager_ssl_errors(self, reply, errors): url = unicode(reply.url().toString()) if self.ignore_ssl_errors: reply.ignoreSslErrors() else: Logger.log('SSL certificate error: %s' % url, level='warning')
def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), plugins_enabled=False, java_enabled=False, plugin_path=[ '/usr/lib/mozilla/plugins', ], download_images=True, qt_debug=False): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Error('Xvfb is required to a ghost run outside ' + 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) if plugin_path: for p in plugin_path: Ghost._app.addLibraryPath(p) self.popup_messages = [] self.page = GhostWebPage(Ghost._app, self) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute( QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) self.page.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, download_images) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: class MyQWebView(QtWebKit.QWebView): def sizeHint(self): return QSize(*viewport_size) self.webview = MyQWebView() if plugins_enabled: self.webview.settings().setAttribute( QtWebKit.QWebSettings.PluginsEnabled, True) if java_enabled: self.webview.settings().setAttribute( QtWebKit.QWebSettings.JavaEnabled, True) self.webview.setPage(self.page) self.webview.show() else: self.webview = None
def cookiesForUrl(self, qurl): cookies = QNetworkCookieJar.cookiesForUrl(self, qurl) #for i in cookies: # info = get_cookie_info(i) # print "------------>> %(domain)s " % info return cookies
def __init__(self, user_agent=default_user_agent, operate_timeout=10, loading_timeout=60, log_level=logging.WARNING, display=False, viewport_size=(1024, 768), accept_language='en,*', ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "GRobot"), image_enabled=True, plugins_enabled=False, java_enabled=False, javascript_enabled=True, plugin_path=None, develop=False, proxy=None, sleep=0.5, jquery_namespace='GRobot'): """GRobot manages a QWebPage. @param user_agent: The default User-Agent header. @param operate_timeout: Operation timeout. @param loading_timeout: The page loading timeout. @param log_level: The optional logging level. @param display: A boolean that tells GRobot to displays UI. @param viewport_size: A tupple that sets initial viewport size. @param accept_language: Set the webkit accept language. @param ignore_ssl_errors: A boolean that forces ignore ssl errors. @param cache_dir: A directory path where to store cache datas. @param image_enabled: Enable images. @param plugins_enabled: Enable plugins (like Flash). @param java_enabled: Enable Java JRE. @param javascript_enabled: Enable Javascript. @param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins']) @param develop: Enable the Webkit Inspector. @param proxy: Set a Socks5,HTTP{S} Proxy @param sleep: Sleep `sleep` second,after operate @param jquery_namespace: Set the jQuery namespace. """ GRobot.exit_lock.acquire() if GRobot._kill_loop: gevent.kill(GRobot._kill_loop) GRobot._kill_loop=None logger.setLevel(log_level) plugin_path = plugin_path or ['/usr/lib/mozilla/plugins', ] GRobot._liveRobot += 1 self.develop = develop self.inspector = None self.plugin = False self.exitLoop = False self._deleted = False self.set_proxy(proxy) self.sleep = sleep self.jquery_namespace = jquery_namespace self.popup_messages = None self.accept_language = accept_language self._loaded = True self._confirm_expected = None self._prompt_expected = None self._upload_file = None self._alert = None self.http_resources = [] self.user_agent = user_agent self.loading_timeout = loading_timeout self.operate_timeout = operate_timeout self.ignore_ssl_errors = ignore_ssl_errors if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ \ and not hasattr(GRobot, 'xvfb'): try: os.environ['DISPLAY'] = ':99' GRobot.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a GRobot run oustside ' + \ 'an X instance') self.display = display if not GRobot._app: GRobot._app = QApplication.instance() or QApplication(['GRobot']) if plugin_path: for p in plugin_path: GRobot._app.addLibraryPath(p) self.page = GRobotWebPage(self, GRobot._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() #TODO:Think about how to handle the network accessible signal #self.manager.networkAccessibleChanged.connect() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired \ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired \ .connect(self._authenticate) self.main_frame = self.page.mainFrame() self.webview = None self.viewport_size = viewport_size self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() if display else self.webview.hide() self.set_viewport_size(*viewport_size) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.page.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, self.develop) self.enable_image = image_enabled self.enable_javascript = javascript_enabled #always open link in current window instead of new window self.page.setLinkDelegationPolicy(QWebPage.DelegateAllLinks) self.page.linkClicked.connect(self._link_clicked) #start the qt main loop GRobot._loop = QtMainLoop(GRobot._app) GRobot._loop.start() GRobot.exit_lock.release()
class GRobot(object): _loop = None _liveRobot = 0 _app = None _kill_loop=None exit_lock = RLock() def __init__(self, user_agent=default_user_agent, operate_timeout=10, loading_timeout=60, log_level=logging.WARNING, display=False, viewport_size=(1024, 768), accept_language='en,*', ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "GRobot"), image_enabled=True, plugins_enabled=False, java_enabled=False, javascript_enabled=True, plugin_path=None, develop=False, proxy=None, sleep=0.5, jquery_namespace='GRobot'): """GRobot manages a QWebPage. @param user_agent: The default User-Agent header. @param operate_timeout: Operation timeout. @param loading_timeout: The page loading timeout. @param log_level: The optional logging level. @param display: A boolean that tells GRobot to displays UI. @param viewport_size: A tupple that sets initial viewport size. @param accept_language: Set the webkit accept language. @param ignore_ssl_errors: A boolean that forces ignore ssl errors. @param cache_dir: A directory path where to store cache datas. @param image_enabled: Enable images. @param plugins_enabled: Enable plugins (like Flash). @param java_enabled: Enable Java JRE. @param javascript_enabled: Enable Javascript. @param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins']) @param develop: Enable the Webkit Inspector. @param proxy: Set a Socks5,HTTP{S} Proxy @param sleep: Sleep `sleep` second,after operate @param jquery_namespace: Set the jQuery namespace. """ GRobot.exit_lock.acquire() if GRobot._kill_loop: gevent.kill(GRobot._kill_loop) GRobot._kill_loop=None logger.setLevel(log_level) plugin_path = plugin_path or ['/usr/lib/mozilla/plugins', ] GRobot._liveRobot += 1 self.develop = develop self.inspector = None self.plugin = False self.exitLoop = False self._deleted = False self.set_proxy(proxy) self.sleep = sleep self.jquery_namespace = jquery_namespace self.popup_messages = None self.accept_language = accept_language self._loaded = True self._confirm_expected = None self._prompt_expected = None self._upload_file = None self._alert = None self.http_resources = [] self.user_agent = user_agent self.loading_timeout = loading_timeout self.operate_timeout = operate_timeout self.ignore_ssl_errors = ignore_ssl_errors if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ \ and not hasattr(GRobot, 'xvfb'): try: os.environ['DISPLAY'] = ':99' GRobot.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a GRobot run oustside ' + \ 'an X instance') self.display = display if not GRobot._app: GRobot._app = QApplication.instance() or QApplication(['GRobot']) if plugin_path: for p in plugin_path: GRobot._app.addLibraryPath(p) self.page = GRobotWebPage(self, GRobot._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() #TODO:Think about how to handle the network accessible signal #self.manager.networkAccessibleChanged.connect() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired \ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired \ .connect(self._authenticate) self.main_frame = self.page.mainFrame() self.webview = None self.viewport_size = viewport_size self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() if display else self.webview.hide() self.set_viewport_size(*viewport_size) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.page.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, self.develop) self.enable_image = image_enabled self.enable_javascript = javascript_enabled #always open link in current window instead of new window self.page.setLinkDelegationPolicy(QWebPage.DelegateAllLinks) self.page.linkClicked.connect(self._link_clicked) #start the qt main loop GRobot._loop = QtMainLoop(GRobot._app) GRobot._loop.start() GRobot.exit_lock.release() @property def popup_messages(self): return self._popup_messages @popup_messages.setter def popup_messages(self, value): self._popup_messages = str(value) @property def url(self): return str(self.main_frame.url().toString()) def content(self): """Returns current frame HTML as a string.""" return str(self.main_frame.toHtml()) @property def cookies(self): """Returns all cookies.""" return self.cookie_jar.allCookies() @property def enable_image(self): """Disable the page images can speed up page loading. """ return self._enable_image @enable_image.setter def enable_image(self, value): self.page.settings().setAttribute(QWebSettings.AutoLoadImages, value) self._enable_image = value #TODO:It seems not work? # @enable_image.deleter # def enable_image(self): # raise NotImplemented @property def enable_javascript(self): """Disable the page javascript can speed up page loading. """ return self._enable_javascript @enable_javascript.setter def enable_javascript(self, value): self.page.settings().setAttribute(QWebSettings.JavascriptEnabled, value) self._enable_javascript = value def open(self, address, method='get', headers=None, auth=None, body=None, default_popup_response=None): """Opens a web page. @param address: The resource URL. @param method: The Http method. @param headers: An optional dict of extra request hearders. @param auth: An optional tupple of HTTP auth (username, password). @param body: An optional string containing a payload. @param default_popup_response: the default response for any confirm/ alert/prompt popup from the Javascript (replaces the need for the with blocks) """ headers = headers or {} body = body or QByteArray() try: method = getattr(QNetworkAccessManager, "%sOperation" % method.capitalize()) except AttributeError: raise Exception("Invalid http method %s" % method) request = QNetworkRequest(QUrl(address)) request.CacheLoadControl = 0 for header in headers: request.setRawHeader(header, headers[header]) self._auth = auth self._auth_attempt = 0 # Avoids reccursion self.page.mainFrame().load(request, method, body) self._loaded = False if default_popup_response is not None: self._prompt_expected = (default_popup_response, None) self._confirm_expected = (default_popup_response, None) return self.wait_for_page_loaded() def set_viewport_size(self, width, height): """Sets the page viewport size. @param width: An integer that sets width pixel count. @param height: An integer that sets height pixel count. """ if self.display: self.webview.resize(QSize(width, height)) self.page.setViewportSize(QSize(width, height)) def set_proxy(self, proxy=None): """Set the proxy or using system configuration as None,supported socks5 http{s}. @param proxy: Example:socks5://username:[email protected]:7070 """ proxy_type = None if proxy: parse = urlparse(proxy) scheme = parse.scheme hostname = parse.hostname port = parse.port username = parse.username or '' password = parse.password or '' if scheme == 'socks5': proxy_type = QNetworkProxy.Socks5Proxy elif scheme in ('http', 'https'): proxy_type = QNetworkProxy.HttpProxy if proxy_type: self.page.networkAccessManager().setProxy( QNetworkProxy(proxy_type, hostname, port, username, password) ) else: QNetworkProxyFactory.setUseSystemConfiguration(True) def first_element_position(self, selector): try: return self.elements_position(selector)[0] except IndexError: logger.warning("Can't locate selector " + selector) return None def elements_position(self, selector): """Get the position of elements whose match selector @param selector: @return: position of QPoint """ attr, pattern, val = self.parser_selector(selector, attr='identifier') strip = lambda v: v.strip() if pattern: val = locals()[pattern](val) def identifier(query): return id(query) or name(query) def name(query): return css("*[name='%s']" % query) def id(query): return css('#' + query) def link(query): return xpath("//a[text()='%s']" % query.replace("\'", "\\'")) def css(query): result = [] for ele in self.main_frame.findAllElements(query): if not ele.isNull(): result.append(ele.geometry().center()) return result def xpath(query): positions = self.evaluate(""" function GetAbsoluteLocationEx(element) { if ( arguments.length != 1 || element == null ) { return null; } var elmt = element; var offsetTop = elmt.offsetTop; var offsetLeft = elmt.offsetLeft; var offsetWidth = elmt.offsetWidth; var offsetHeight = elmt.offsetHeight; while( elmt = elmt.offsetParent ) { // add this judge if ( elmt.style.position == 'absolute' || elmt.style.position == 'relative' || ( elmt.style.overflow != 'visible' && elmt.style.overflow != '' ) ) { break; } offsetTop += elmt.offsetTop; offsetLeft += elmt.offsetLeft; } return { absoluteTop: offsetTop, absoluteLeft: offsetLeft, offsetWidth: offsetWidth, offsetHeight: offsetHeight }; } result=[]; for (var r = document.evaluate('%s', document, null, 5, null), n; n = r.iterateNext();) { pos=GetAbsoluteLocationEx(n) result.push([pos.absoluteLeft+pos.offsetWidth/2.0,pos.absoluteTop+pos.offsetHeight/2.0]); } result """ % query.replace("\'", "\\'")) return [QPoint(*tuple(x)) for x in positions] return locals()[attr](val) def _move_page_center_to(self, qpoint): size = self.page.viewportSize() self.main_frame.setScrollPosition(qpoint - QPoint(size.width(), size.height()) / 2) def reload(self): """Reload page. @return: """ self.trigger_action('Reload', expect_loading=True) def back(self): self.trigger_action('Back') def forward(self): self.trigger_action('Forward') @can_load_page def trigger_action(self, action): """Trigger QWebPage::WebAction @param action: """ self.page.triggerAction(getattr(QWebPage, action)) def parser_selector(self, selector, attr=None, pattern=None, val=None): index = selector.find('=') if index <= 0: val = selector else: attr = selector[:index] value_ = selector[index + 1:] index = value_.find(':') if index > 0: pattern = value_[:index] val = value_[index + 1:] return attr, pattern, val @can_load_page @have_a_break def click(self, selector): qpoint = self.first_element_position(selector) if qpoint: return self.qpoint_to_tuple(self._click_position(qpoint)) @can_load_page @have_a_break def test(self): return self.qpoint_to_tuple(QPoint(1, 2)) @can_load_page def _click_position(self, qpoint): self._move_page_center_to(qpoint) self.webview.repaint() pos = qpoint - self.main_frame.scrollPosition() self._move_to_position(pos) QTest.mouseClick(self.webview, Qt.LeftButton, pos=pos) gevent.sleep(1) return pos def qpoint_to_tuple(self, qpoint): return qpoint.x(), qpoint.y() @have_a_break def move_to(self, selector): qpoint = self.first_element_position(selector) if qpoint: self._move_to_position(qpoint) return qpoint_to_tuple(qpoint) def move_at(self, x, y): return self._move_to_position(QPoint(x, y)) def _move_to_position(self, qpoint): QTest.mouseMove(self.webview, pos=qpoint) return qpoint @can_load_page @have_a_break def click_at(self, x, y): return self._click_position(QPoint(x, y)) @have_a_break def key_clicks(self, selector, text): if selector: self.click(selector) QTest.keyClicks(self.webview, text, delay=50) @have_a_break def type(self, selector, text): position = self.click(selector) ele = self._hit_element_from(QPoint(*position)) ele.setFocus() ele.evaluateJavaScript( """ core.events.setValue(this, '%s') """ % (text.replace("\n", "\\n").replace("\'", "\\'")) ) logger.debug('type %s %s' % (selector, text)) def _hit_element_from(self, position): return self.main_frame.hitTestContent(position).element() def first_element(self, selector): position = self.first_element_position(selector) if position: return self.main_frame.hitTestContent(position).element(), position def wait_forever(self): self.wait_for(lambda: False, time_for_stop=-1) @have_a_break def check(self, selector, checked=True): ele, position = self.first_element(selector) if ele and ele.tagName() == 'INPUT': if ele.attribute('type') in ['checkbox', 'radio']: ele_checked = ele.attribute('checked') == 'checked' or False if ele_checked != checked: self._click_position(position) else: raise ValueError("%s is not a checkbox or radio" % selector) @have_a_break def select(self, selector, value): def _select(query, select_by, select): select.evaluateJavaScript(""" triggerEvent(this, 'focus', false); var changed = false; var optionToSelect = '%s'; for (var i = 0; i < this.options.length; i++) { var option = this.options[i]; if (option.selected && option.%s != optionToSelect) { option.selected = false; changed = true; } else if (!option.selected && option.%s == optionToSelect) { option.selected = true; changed = true; } } if (changed) { triggerEvent(this, 'change', true); } """ % ( query.replace("\'", "\\'"), select_by, select_by)) def _add_selection(query, select_by, select, selected): select.evaluateJavaScript(""" triggerEvent(this, 'focus', false); var optionToSelect = '%s'; for (var i = 0; i < this.options.length; i++) { var option = this.options[i]; if (option.%s == optionToSelect) { option.selected = %s; triggerEvent(this, 'change', true); } } """ % ( query.replace("\'", "\\'"), select_by, selected and 'true' or 'false')) ele, position = self.first_element(selector) if ele and ele.tagName() == 'SELECT': ele.setFocus() if ele.attribute('multiple') == 'multiple': assert isinstance(value, list) for value_, selected in value: attr, pattern, val = self.parser_selector(value_, attr='text') _add_selection(val, attr, ele, selected) else: attr, pattern, val = self.parser_selector(value, attr='text') _select(val, attr, ele) def choose_file(self, selector, file): self._upload_file = file self.click(selector) self._upload_file = None def capture(self, selector=None): """Capture the images of selector. @param selector: Css selector. @return: Images """ elements = self.main_frame.documentElement().findAll(selector) imgs = [] for element in elements: geo = element.geometry() img = QImage(geo.width(), geo.height(), QImage.Format_ARGB32) painter = QPainter(img) element.render(painter) painter.end() imgs.append(img) return imgs def capture_to(self, path, selector=None): """Capture the images of selector to files. @param path: File path with index suffix. @param selector: Css selector. @return: The paths of saving. """ _, ext = os.path.splitext(path) ext = ext[1:] imgs = self.capture(selector) result = [] for index, img in enumerate(imgs): filepath = '%s.%s' % (path, index) if img.save(filepath, ext.upper()): result.append(filepath) return result def capture_to_buf(self, selector=None): """capture the images of selector to StringIO @param selector: Css selector. @return: The StringIO list. """ images = self.capture(selector) result = [] for image in images: ba = QByteArray() buf = QBuffer(ba) buf.open(QIODevice.ReadWrite) image.save(buf, 'jpg') stream = StringIO(str(buf.buffer())) result.append(stream) return result @can_load_page def evaluate(self, script): """Evaluates script in page frame. @param script: The script to evaluate. """ result = self.main_frame.evaluateJavaScript("%s" % script) # if isinstance(result,QString): # result=unicode(result) return result def evaluate_js_file(self, path, encoding='utf-8'): """Evaluates javascript file at given path in current frame. Raises native IOException in case of invalid file. @param path: The path of the file. @param encoding: The file's encoding. """ self.evaluate(codecs.open(path, encoding=encoding).read()) def __del__(self): """Depend on the CG of Python. """ self.exit() def delete_cookies(self): """Deletes all cookies.""" self.cookie_jar.setAllCookies([]) def exists(self, selector): """Checks if element exists for given selector. @param string: The element selector. """ return not self.main_frame.findFirstElement(selector).isNull() #TODO: Still not work. # def remove_css(self): # """Remore the css,speed up page loading. # # @return: # """ # # return self.evaluate("""var targetelement="link";//determine element type to create nodelist from # var targetattr="href"//determine corresponding attribute to test for # var allsuspects=document.getElementsByTagName(targetelement) # for (var i=allsuspects.length; i>=0; i--){ //search backwards within nodelist for matching elements to remove # if (allsuspects[i] && allsuspects[i].getAttribute(targetattr)!=null ) # allsuspects[i].parentNode.removeChild(allsuspects[i]); //remove element by calling parentNode.removeChild() # } # """) def filter_resources(self, pattern): """Filter resources with pattern. @param pattern: Match pattern. @param resources: @return: @raise: """ if isinstance(pattern, str): is_match = lambda x: pattern == x elif isinstance(pattern, _pattern_type): is_match = lambda x: pattern.match(x) elif hasattr(pattern, '__call__'): is_match = pattern else: raise TypeError('pattern must be one of str,re.compile,callable') return filter(lambda x: is_match(x.request_url), self.http_resources)[:] def save(self, path): """Save current page content to the path. @param path: The path to save. """ f = open(path, 'w') f.write(self.content().encode('utf-8')) f.close() def global_exists(self, global_name): """Checks if javascript global exists. @param global_name: The name of the global. """ return self.evaluate('!(typeof %s === "undefined");' % global_name) def load_cookies( self, cookie_storage, keep_old=False ): """load from cookielib's CookieJar or Set-Cookie3 format text file. @param cookie_storage: file location string on disk or CookieJar instance. @param keep_old: Don't reset, keep cookies not overridden. """ def toQtCookieJar( PyCookieJar, QtCookieJar ): allCookies = QtCookieJar.cookies if keep_old else [] for pc in PyCookieJar: qc = toQtCookie(pc) allCookies.append(qc) QtCookieJar.setAllCookies(allCookies) def toQtCookie(PyCookie): qc = QNetworkCookie(PyCookie.name, PyCookie.value) qc.setSecure(PyCookie.secure) if PyCookie.path_specified: qc.setPath(PyCookie.path) if PyCookie.domain != "": qc.setDomain(PyCookie.domain) if PyCookie.expires != 0: t = QDateTime() t.setTime_t(PyCookie.expires) qc.setExpirationDate(t) # not yet handled(maybe less useful): # py cookie.rest / QNetworkCookie.setHttpOnly() return qc if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) cj.load() toQtCookieJar(cj, self.cookie_jar) elif cookie_storage.__class__.__name__.endswith('CookieJar'): toQtCookieJar(cookie_storage, self.cookie_jar) else: raise ValueError('unsupported cookie_storage type.') def save_cookies(self, cookie_storage): """Save to cookielib's CookieJar or Set-Cookie3 format text file. @param cookie_storage: file location string or CookieJar instance. """ def toPyCookieJar(QtCookieJar, PyCookieJar): for c in QtCookieJar.allCookies(): PyCookieJar.set_cookie(toPyCookie(c)) def toPyCookie(QtCookie): port = None port_specified = False secure = QtCookie.isSecure() name = str(QtCookie.name()) value = str(QtCookie.value()) v = str(QtCookie.path()) path_specified = bool(v != "") path = v if path_specified else None v = str(QtCookie.domain()) domain_specified = bool(v != "") domain = v domain_initial_dot = v.startswith('.') if domain_specified else None v = int(QtCookie.expirationDate().toTime_t()) # Long type boundary on 32bit platfroms; avoid ValueError expires = 2147483647 if v > 2147483647 else v rest = {} discard = False return Cookie(0, name, value, port, port_specified, domain , domain_specified, domain_initial_dot, path, path_specified , secure, expires, discard, None, None, rest) if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) toPyCookieJar(self.cookie_jar, cj) cj.save() elif cookie_storage.__class__.__name__.endswith('CookieJar'): toPyCookieJar(self.cookie_jar, cookie_storage) else: raise ValueError('unsupported cookie_storage type.') def wait_for_confirm(self, confirm=True, callback=None): """Statement that tells GRobot how to deal with javascript confirm(). @param confirm: A bollean that confirm. @param callable: A callable that returns a boolean for confirmation. """ self._robot._confirm_expected = (confirm, callback) self._robot.wait_for(lambda: self._robot._confirm_expected is None) return self.popup_messages def wait_for_text(self, text, time_for_stop=None): """Waits until given text appear on main frame. @param text: The text to wait for. @return: """ logger.debug("Wait for text %s" % text) self.wait_for(lambda: text in self.content(), "Can\'t find '%s' in current frame" % text, time_for_stop=time_for_stop) return self.wait_for_page_loaded() def wait_for_xpath(self, expression, time_for_stop=None): self.wait_for(lambda: XPath(self.content()).execute(expression), "Can't find xpath=%s in current frame" % expression, time_for_stop=time_for_stop) return self.wait_for_page_loaded() def wait_for_selector(self, selector): """Waits until selector match an element on the frame. @param selector: The selector to wait for. """ self.wait_for(lambda: self.exists(selector), 'Can\'t find element matching "%s"' % selector) def wait_for_page_loaded(self, time_for_stop=None): """Waits until page is loaded, assumed that a page as been requested. """ return self.wait_for(lambda: self._loaded, 'Unable to load requested page', time_for_stop=time_for_stop) def wait_for(self, condition, timeout_message='', time_for_stop=None): """Waits until condition is True. @param condition: A callable that returns the condition. @param timeout_message: The exception message on timeout.-1 means never timeout. """ if self._loaded: time_for_stop = time_for_stop or self.operate_timeout else: time_for_stop = time_for_stop or self.loading_timeout started_at = time.time() while not condition(): if time_for_stop != -1 and time.time() > (started_at + time_for_stop): if self._loaded: raise OperateTimeout(timeout_message) else: # raise LoadingTimeout, timeout_message self.trigger_action('Stop') #QWebPage::Stop self._loaded = True logger.warning("Page loading timeout.Force to stop the page") break gevent.sleep(2) def wait_for_alert(self): """Waits for main frame alert(). """ self.wait_for(lambda: self._alert is not None, 'User has not been alerted.') msg, self._alert = self._alert, None return msg def _release_last_resources(self): """Releases last loaded resources. :return: The released resources. """ last_resources, self.http_resources = self.http_resources[:], [] return last_resources def _page_loaded(self, success): if self.develop and self.display: if self.inspector is None: self.inspector = QWebInspector() self.inspector.setPage(self.page) self.inspector.show() scripts = [ 'atoms.js', 'htmlutils.js', ] if self.jquery_namespace: scripts.append('jquery-1.9.1.min.js', ) for script in scripts: self.evaluate_js_file(os.path.dirname(__file__) + '/javascripts/' + script) if self.jquery_namespace: self.evaluate("%s=jQuery.noConflict();" % self.jquery_namespace) self._loaded = True # self.cache.clear() logger.debug("Page load finished") def _page_load_started(self): logger.debug("Start load page") self._loaded = False def _unsupported_content(self, reply): """Adds an HttpResource object to http_resources with unsupported content. @param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache, reply.readAll())) def _link_clicked(self, href): """Contorl the page link clicked event,forbid open new window. @param href: The href attribute of a tag. """ self.main_frame.load(href) def _request_ended(self, reply): """Adds an HttpResource object to http_resources. @param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache)) def _authenticate(self, mix, authenticator): """Called back on basic / proxy http auth. @param mix: The QNetworkReply or QNetworkProxy object. @param authenticator: The QAuthenticator object. """ if self._auth_attempt == 0: username, password = self._auth authenticator.setUser(username) authenticator.setPassword(password) self._auth_attempt += 1 def _on_manager_ssl_errors(self, reply, errors): """Ingore all the ssl error @param reply: @param errors: """ url = str(reply.url().toString()) if self.ignore_ssl_errors: reply.ignoreSslErrors() else: logger.warning('SSL certificate error: %s' % url) def exit(self): """Destroy the Qt main event loop. """ GRobot.exit_lock.acquire() if not self._deleted: if self.inspector: self.inspector.close() sip.delete(self.inspector) if self.display: self.webview.close() sip.delete(self.webview) if self.page and not sip.isdeleted(self.page): sip.delete(self.page) GRobot._liveRobot -= 1 if GRobot._liveRobot == 0 and GRobot._loop is not None: GRobot._kill_loop=gevent.spawn_later(20,self.kill_loop) self._deleted = True GRobot.exit_lock.release() def kill_loop(self): GRobot._loop.stop() GRobot._loop = None GRobot._app = None if hasattr(self, 'xvfb'): GRobot.xvfb.terminate()
def _handle_request_cookies(self, request): jar = QNetworkCookieJar() self.setCookieJar(jar) cookiejar = self._getWebPageAttribute(request, "cookiejar") if cookiejar is not None: cookiejar.update_cookie_header(request)
class ShoutCastForm(PluginBase.PluginBase): '''Grab Shoutcast streams and save them as "bookmarks" - and play them on the currently selected server. General shoutcast information is not preserved between runs. Also, the shoutcast server/API is pretty lame so timeouts actually occur quite frequently. ''' moduleName = '&Shoutcast' moduleIcon = "network-workgroup" def load(self): pass def event(self, event): if event.type() == QEvent.Paint: if not hasattr(self, 'webView'): self._load() self.event = super(ShoutCastForm, self).event return False def _load(self): self.cookie = QNetworkCookie('Settings', 'Player~others|Bandwidth~ALL|Codec~ALL') self.cookie.setDomain('.shoutcast.com') self.cookie.setExpirationDate(QDateTime()) self.cookie.setPath('/') self.webView = QWebView(self) self.webPage = self.webView.page() self.cookiejar = QNetworkCookieJar() self.cookiejar.setAllCookies([self.cookie]) self.webPage.networkAccessManager().setCookieJar(self.cookiejar) self.layout = QVBoxLayout(self) self.layout.addWidget(self.webView) self.webView.load(QUrl(HOMEURL)) self.webPage.setLinkDelegationPolicy(QWebPage.DelegateExternalLinks) self.connect(self.webPage, SIGNAL('linkClicked(const QUrl&)'), self._processLink) def _processLink(self, url): if url.host() == TUNEIN: self._playStation(url.toString()) else: self.webView.load(url) self.webView.show() def _playStation(self, url): data = self._retreivePLS(url) adrlist = self._parsePLS(data) self.mpdclient.send('command_list_ok_begin') try: for address in adrlist: self.mpdclient.send('add', (address,)) finally: self.mpdclient.send('command_list_end') def _retreivePLS(self, url): conn = httplib.HTTPConnection(TUNEIN) conn.request("GET", TUNEINFORMAT % url.split('=')[-1]) resp = conn.getresponse() if resp.status == 200: return resp.read().split('\n') else: raise httplib.HTTPException('Got bad status code.') def _parsePLS(self, data): adrlist = [] state = '' while data: line = data.pop(0) if state == '' and line == '[playlist]': state = 'playlist' elif state == 'playlist': if '=' in line: key, value = line.split('=', 1) if key.startswith('File'): adrlist.append(value) else: raise httplib.HTTPException('Encountered error during parsing of the playlist.') return adrlist
class GRobot(object): _loop = None _liveRobot = 0 _app = None exit_lock = RLock() def __init__(self, user_agent=default_user_agent, operate_timeout=10, loading_timeout=60, log_level=logging.WARNING, display=False, viewport_size=(1024, 768), accept_language='en,*', ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "GRobot"), image_enabled=True, plugins_enabled=False, java_enabled=False, javascript_enabled=True, plugin_path=None, develop=False, proxy=None, sleep=0.5, jquery_namespace='GRobot'): """GRobot manages a QWebPage. @param user_agent: The default User-Agent header. @param operate_timeout: Operation timeout. @param loading_timeout: The page loading timeout. @param log_level: The optional logging level. @param display: A boolean that tells GRobot to displays UI. @param viewport_size: A tupple that sets initial viewport size. @param accept_language: Set the webkit accept language. @param ignore_ssl_errors: A boolean that forces ignore ssl errors. @param cache_dir: A directory path where to store cache datas. @param image_enabled: Enable images. @param plugins_enabled: Enable plugins (like Flash). @param java_enabled: Enable Java JRE. @param javascript_enabled: Enable Javascript. @param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins']) @param develop: Enable the Webkit Inspector. @param proxy: Set a Socks5,HTTP{S} Proxy @param sleep: Sleep `sleep` second,after operate @param jquery_namespace: Set the jQuery namespace. """ GRobot.exit_lock.acquire() logger.setLevel(log_level) plugin_path = plugin_path or ['/usr/lib/mozilla/plugins', ] GRobot._liveRobot += 1 self.develop = develop self.inspector = None self.plugin = False self.exitLoop = False self.set_proxy(proxy) self.sleep = sleep self.jquery_namespace = jquery_namespace self.popup_messages = None self.accept_language = accept_language self._loaded = True self._confirm_expected = None self._prompt_expected = None self._upload_file = None self._alert = None self.http_resources = [] self.user_agent = user_agent self.loading_timeout = loading_timeout self.operate_timeout = operate_timeout self.ignore_ssl_errors = ignore_ssl_errors if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ \ and not hasattr(GRobot, 'xvfb'): try: os.environ['DISPLAY'] = ':99' GRobot.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a GRobot run oustside ' + \ 'an X instance') self.display = display if not GRobot._app: GRobot._app = QApplication.instance() or QApplication(['GRobot']) if plugin_path: for p in plugin_path: GRobot._app.addLibraryPath(p) self.page = GRobotWebPage(self, GRobot._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() #TODO:Think about how to handle the network accessible signal #self.manager.networkAccessibleChanged.connect() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired \ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired \ .connect(self._authenticate) self.main_frame = self.page.mainFrame() self.webview = None self.viewport_size = viewport_size self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() if display else self.webview.hide() self.set_viewport_size(*viewport_size) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.page.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, self.develop) self.enable_image = image_enabled self.enable_javascript = javascript_enabled #always open link in current window instead of new window self.page.setLinkDelegationPolicy(QWebPage.DelegateAllLinks) self.page.linkClicked.connect(self._link_clicked) #start the qt main loop GRobot._loop = QtMainLoop(GRobot._app) GRobot._loop.start() GRobot.exit_lock.release() @property def popup_messages(self): return self._popup_messages @popup_messages.setter def popup_messages(self, value): self._popup_messages = unicode(value) @property def url(self): return unicode(self.main_frame.url().toString()) @property def content(self): """Returns current frame HTML as a string.""" return unicode(self.page.currentFrame().toHtml()) @property def cookies(self): """Returns all cookies.""" return self.cookie_jar.allCookies() @property def enable_image(self): """Disable the page images can speed up page loading. """ return self._enable_image @enable_image.setter def enable_image(self, value): self.page.settings().setAttribute(QWebSettings.AutoLoadImages, value) self._enable_image = value #TODO:It seems not work? # @enable_image.deleter # def enable_image(self): # raise NotImplemented @property def enable_javascript(self): """Disable the page javascript can speed up page loading. """ return self._enable_javascript @enable_javascript.setter def enable_javascript(self, value): self.page.settings().setAttribute(QWebSettings.JavascriptEnabled, value) self._enable_javascript = value def open(self, address, method='get', headers=None, auth=None, body=None, default_popup_response=None): """Opens a web page. @param address: The resource URL. @param method: The Http method. @param headers: An optional dict of extra request hearders. @param auth: An optional tupple of HTTP auth (username, password). @param body: An optional string containing a payload. @param default_popup_response: the default response for any confirm/ alert/prompt popup from the Javascript (replaces the need for the with blocks) """ headers = headers or {} body = body or QByteArray() try: method = getattr(QNetworkAccessManager, "%sOperation" % method.capitalize()) except AttributeError: raise Exception("Invalid http method %s" % method) request = QNetworkRequest(QUrl(address)) request.CacheLoadControl = 0 for header in headers: request.setRawHeader(header, headers[header]) self._auth = auth self._auth_attempt = 0 # Avoids reccursion self.page.mainFrame().load(request, method, body) self._loaded = False if default_popup_response is not None: self._prompt_expected = (default_popup_response, None) self._confirm_expected = (default_popup_response, None) return self.wait_for_page_loaded() def set_viewport_size(self, width, height): """Sets the page viewport size. @param width: An integer that sets width pixel count. @param height: An integer that sets height pixel count. """ if self.display: self.webview.resize(QSize(width, height)) self.page.setViewportSize(QSize(width, height)) def set_proxy(self, proxy=None): """Set the proxy or using system configuration as None,supported socks5 http{s}. @param proxy: Example:socks5://username:[email protected]:7070 """ proxy_type = None if proxy: parse = urlparse(proxy) scheme = parse.scheme hostname = parse.hostname port = parse.port username = parse.username or '' password = parse.password or '' if scheme == 'socks5': proxy_type = QNetworkProxy.Socks5Proxy elif scheme in ('http', 'https'): proxy_type = QNetworkProxy.HttpProxy if proxy_type: self.page.networkAccessManager().setProxy( QNetworkProxy(proxy_type, hostname, port, username, password) ) else: QNetworkProxyFactory.setUseSystemConfiguration(True) def first_element_position(self, selector): try: return self.elements_position(selector)[0] except IndexError: logger.warning("Can't locate selector " + selector) return None def elements_position(self, selector): """Get the position of elements whose match selector @param selector: @return: position of QPoint """ attr, pattern, val = self.parser_selector(selector, attr='identifier') strip = lambda v: v.strip() if pattern: val = locals()[pattern](val) def identifier(query): return id(query) or name(query) def name(query): return css("*[name='%s']" % query) def id(query): return css('#' + query) def link(query): return xpath(u"//a[@text()='%s']" % query.replace("\'", "\\'")) def css(query): result = [] for ele in self.main_frame.findAllElements(query): if not ele.isNull(): result.append(ele.geometry().center()) return result def xpath(query): positions = self.evaluate(u""" function GetAbsoluteLocationEx(element) { if ( arguments.length != 1 || element == null ) { return null; } var elmt = element; var offsetTop = elmt.offsetTop; var offsetLeft = elmt.offsetLeft; var offsetWidth = elmt.offsetWidth; var offsetHeight = elmt.offsetHeight; while( elmt = elmt.offsetParent ) { // add this judge if ( elmt.style.position == 'absolute' || elmt.style.position == 'relative' || ( elmt.style.overflow != 'visible' && elmt.style.overflow != '' ) ) { break; } offsetTop += elmt.offsetTop; offsetLeft += elmt.offsetLeft; } return { absoluteTop: offsetTop, absoluteLeft: offsetLeft, offsetWidth: offsetWidth, offsetHeight: offsetHeight }; } result=[]; for (var r = document.evaluate('%s', document, null, 5, null), n; n = r.iterateNext();) { pos=GetAbsoluteLocationEx(n) result.push([pos.absoluteLeft+pos.offsetWidth/2.0,pos.absoluteTop+pos.offsetHeight/2.0]); } result """ % query.replace("\'", "\\'")) return map(lambda x: QPoint(*tuple(x)), positions) return locals()[attr](val) def _move_page_center_to(self, qpoint): size = self.page.viewportSize() self.main_frame.setScrollPosition(qpoint - QPoint(size.width(), size.height()) / 2) def reload(self): """Reload page. @return: """ self.trigger_action('Reload', expect_loading=True) def back(self): self.trigger_action('Back') def forward(self): self.trigger_action('Forward') @can_load_page def trigger_action(self, action): """Trigger QWebPage::WebAction @param action: """ self.page.triggerAction(getattr(QWebPage, action)) def parser_selector(self, selector, attr=None, pattern=None, val=None): index = selector.find('=') if index <= 0: val = selector else: attr = selector[:index] value_ = selector[index + 1:] index = value_.find(':') if index > 0: pattern = value_[:index] val = value_[index + 1:] return attr, pattern, val @can_load_page @have_a_break def click(self, selector): qpoint = self.first_element_position(selector) if qpoint: return self._click_position(qpoint) @can_load_page def _click_position(self, qpoint): self._move_page_center_to(qpoint) self.webview.repaint() pos = qpoint - self.main_frame.scrollPosition() self._move_to_position(pos) QTest.mouseClick(self.webview, Qt.LeftButton, pos=pos) gevent.sleep(1) return pos def qpoint_to_tuple(self, qpoint): return qpoint.x(), qpoint.y() @have_a_break def move_to(self, selector): qpoint = self.first_element_position(selector) if qpoint: self._move_to_position(qpoint) return qpoint_to_tuple(qpoint) def move_at(self, x, y): self._move_to_position(QPoint(x, y)) def _move_to_position(self, qpoint): QTest.mouseMove(self.webview, pos=qpoint) return qpoint @have_a_break def click_at(self, x, y): self._click_position(QPoint(x, y)) @have_a_break def key_clicks(self, selector, text): if selector: self.click(selector) QTest.keyClicks(self.webview, text, delay=50) @have_a_break def type(self, selector, text): position = self.click(selector) ele = self._hit_element_from(position) ele.setFocus() ele.evaluateJavaScript( u""" core.events.setValue(this, '%s') """ % (text.replace("\n", "\\n").replace("\'", "\\'")) ) logger.debug('type %s %s' % (selector, text)) def _hit_element_from(self, position): return self.main_frame.hitTestContent(position).element() def first_element(self, selector): position = self.first_element_position(selector) if position: return self.main_frame.hitTestContent(position).element(), position def wait_forever(self): self.wait_for(lambda: False, time_for_stop=-1) @have_a_break def check(self, selector, checked=True): ele, position = self.first_element(selector) if ele and ele.tagName() == 'INPUT': if ele.attribute('type') in ['checkbox', 'radio']: ele_checked = ele.attribute('checked') == 'checked' or False if ele_checked != checked: self._click_position(position) else: raise ValueError, "%s is not a checkbox or radio" % selector @have_a_break def select(self, selector, value): def _select(query, select_by, select): select.evaluateJavaScript(u""" triggerEvent(this, 'focus', false); var changed = false; var optionToSelect = '%s'; for (var i = 0; i < this.options.length; i++) { var option = this.options[i]; if (option.selected && option.%s != optionToSelect) { option.selected = false; changed = true; } else if (!option.selected && option.%s == optionToSelect) { option.selected = true; changed = true; } } if (changed) { triggerEvent(this, 'change', true); } """ % ( query.replace("\'", "\\'"), select_by, select_by)) def _add_selection(query, select_by, select, selected): select.evaluateJavaScript(u""" triggerEvent(this, 'focus', false); var optionToSelect = '%s'; for (var i = 0; i < this.options.length; i++) { var option = this.options[i]; if (option.%s == optionToSelect) { option.selected = %s; triggerEvent(this, 'change', true); } } """ % ( query.replace("\'", "\\'"), select_by, selected and 'true' or 'false')) ele, position = self.first_element(selector) if ele and ele.tagName() == 'SELECT': ele.setFocus() if ele.attribute('multiple') == 'multiple': assert isinstance(value, list) for value_, selected in value: attr, pattern, val = self.parser_selector(value_, attr='text') _add_selection(val, attr, ele, selected) else: attr, pattern, val = self.parser_selector(value, attr='text') _select(val, attr, ele) def choose_file(self, selector, file): self._upload_file = file self.click(selector) self._upload_file = None def capture(self, selector=None): """Capture the images of selector. @param selector: Css selector. @return: Images """ elements = self.main_frame.documentElement().findAll(selector) imgs = [] for element in elements: geo = element.geometry() img = QImage(geo.width(), geo.height(), QImage.Format_ARGB32) painter = QPainter(img) element.render(painter) painter.end() imgs.append(img) return imgs def capture_to(self, path, selector=None): """Capture the images of selector to files. @param path: File path with index suffix. @param selector: Css selector. @return: The paths of saving. """ _, ext = os.path.splitext(path) ext = ext[1:] imgs = self.capture(selector) result = [] for index, img in enumerate(imgs): filepath = '%s.%s' % (path, index) if img.save(filepath, ext.upper()): result.append(filepath) return result def capture_to_buf(self, selector=None): """capture the images of selector to StringIO @param selector: Css selector. @return: The StringIO list. """ images = self.capture(selector) result = [] for image in images: ba = QByteArray() buf = QBuffer(ba) buf.open(QIODevice.ReadWrite) image.save(buf, 'jpg') stream = StringIO(str(buf.buffer())) result.append(stream) return result @can_load_page def evaluate(self, script): """Evaluates script in page frame. @param script: The script to evaluate. """ result = self.main_frame.evaluateJavaScript("%s" % script) # if isinstance(result,QString): # result=unicode(result) return result def evaluate_js_file(self, path, encoding='utf-8'): """Evaluates javascript file at given path in current frame. Raises native IOException in case of invalid file. @param path: The path of the file. @param encoding: The file's encoding. """ self.evaluate(codecs.open(path, encoding=encoding).read()) def __del__(self): """Depend on the CG of Python. """ self._exit() def delete_cookies(self): """Deletes all cookies.""" self.cookie_jar.setAllCookies([]) def exists(self, selector): """Checks if element exists for given selector. @param string: The element selector. """ return not self.main_frame.findFirstElement(selector).isNull() #TODO: Still not work. # def remove_css(self): # """Remore the css,speed up page loading. # # @return: # """ # # return self.evaluate("""var targetelement="link";//determine element type to create nodelist from # var targetattr="href"//determine corresponding attribute to test for # var allsuspects=document.getElementsByTagName(targetelement) # for (var i=allsuspects.length; i>=0; i--){ //search backwards within nodelist for matching elements to remove # if (allsuspects[i] && allsuspects[i].getAttribute(targetattr)!=null ) # allsuspects[i].parentNode.removeChild(allsuspects[i]); //remove element by calling parentNode.removeChild() # } # """) def filter_resources(self, pattern): """Filter resources with pattern. @param pattern: Match pattern. @param resources: @return: @raise: """ if isinstance(pattern, basestring): is_match = lambda x: pattern == x elif isinstance(pattern, _pattern_type): is_match = lambda x: pattern.match(x) elif hasattr(pattern, '__call__'): is_match = pattern else: raise TypeError, 'pattern must be one of str,re.compile,callable' return filter(lambda x: is_match(x.request_url), self.http_resources)[:] def save(self, path): """Save current page content to the path. @param path: The path to save. """ f = open(path, 'w') f.write(self.content.encode('utf-8')) f.close() def global_exists(self, global_name): """Checks if javascript global exists. @param global_name: The name of the global. """ return self.evaluate('!(typeof %s === "undefined");' % global_name) def load_cookies( self, cookie_storage, keep_old=False ): """load from cookielib's CookieJar or Set-Cookie3 format text file. @param cookie_storage: file location string on disk or CookieJar instance. @param keep_old: Don't reset, keep cookies not overridden. """ def toQtCookieJar( PyCookieJar, QtCookieJar ): allCookies = QtCookieJar.cookies if keep_old else [] for pc in PyCookieJar: qc = toQtCookie(pc) allCookies.append(qc) QtCookieJar.setAllCookies(allCookies) def toQtCookie(PyCookie): qc = QNetworkCookie(PyCookie.name, PyCookie.value) qc.setSecure(PyCookie.secure) if PyCookie.path_specified: qc.setPath(PyCookie.path) if PyCookie.domain != "": qc.setDomain(PyCookie.domain) if PyCookie.expires != 0: t = QDateTime() t.setTime_t(PyCookie.expires) qc.setExpirationDate(t) # not yet handled(maybe less useful): # py cookie.rest / QNetworkCookie.setHttpOnly() return qc if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) cj.load() toQtCookieJar(cj, self.cookie_jar) elif cookie_storage.__class__.__name__.endswith('CookieJar'): toQtCookieJar(cookie_storage, self.cookie_jar) else: raise ValueError, 'unsupported cookie_storage type.' def save_cookies(self, cookie_storage): """Save to cookielib's CookieJar or Set-Cookie3 format text file. @param cookie_storage: file location string or CookieJar instance. """ def toPyCookieJar(QtCookieJar, PyCookieJar): for c in QtCookieJar.allCookies(): PyCookieJar.set_cookie(toPyCookie(c)) def toPyCookie(QtCookie): port = None port_specified = False secure = QtCookie.isSecure() name = str(QtCookie.name()) value = str(QtCookie.value()) v = str(QtCookie.path()) path_specified = bool(v != "") path = v if path_specified else None v = str(QtCookie.domain()) domain_specified = bool(v != "") domain = v domain_initial_dot = v.startswith('.') if domain_specified else None v = long(QtCookie.expirationDate().toTime_t()) # Long type boundary on 32bit platfroms; avoid ValueError expires = 2147483647 if v > 2147483647 else v rest = {} discard = False return Cookie(0, name, value, port, port_specified, domain , domain_specified, domain_initial_dot, path, path_specified , secure, expires, discard, None, None, rest) if cookie_storage.__class__.__name__ == 'str': cj = LWPCookieJar(cookie_storage) toPyCookieJar(self.cookie_jar, cj) cj.save() elif cookie_storage.__class__.__name__.endswith('CookieJar'): toPyCookieJar(self.cookie_jar, cookie_storage) else: raise ValueError, 'unsupported cookie_storage type.' def wait_for_confirm(self, confirm=True, callback=None): """Statement that tells GRobot how to deal with javascript confirm(). @param confirm: A bollean that confirm. @param callable: A callable that returns a boolean for confirmation. """ self._robot._confirm_expected = (confirm, callback) self._robot.wait_for(lambda: self._robot._confirm_expected is None) return self.popup_messages def wait_for_text(self, text, time_for_stop=None): """Waits until given text appear on main frame. @param text: The text to wait for. @return: """ logger.debug("Wait for text %s" % text) self.wait_for(lambda: text in self.content, "Can\'t find '%s' in current frame" % text, time_for_stop=time_for_stop) return self.wait_for_page_loaded() def wait_for_xpath(self, expression, time_for_stop=None): self.wait_for(lambda: XPath(self.content).execute(expression), "Can't find xpath=%s in current frame" % expression, time_for_stop=time_for_stop) return self.wait_for_page_loaded() def wait_for_selector(self, selector): """Waits until selector match an element on the frame. @param selector: The selector to wait for. """ self.wait_for(lambda: self.exists(selector), 'Can\'t find element matching "%s"' % selector) def wait_for_page_loaded(self, time_for_stop=None): """Waits until page is loaded, assumed that a page as been requested. """ return self.wait_for(lambda: self._loaded, 'Unable to load requested page', time_for_stop=time_for_stop) def wait_for(self, condition, timeout_message='', time_for_stop=None): """Waits until condition is True. @param condition: A callable that returns the condition. @param timeout_message: The exception message on timeout.-1 means never timeout. """ if self._loaded: time_for_stop = time_for_stop or self.operate_timeout else: time_for_stop = time_for_stop or self.loading_timeout started_at = time.time() while not condition(): if time_for_stop != -1 and time.time() > (started_at + time_for_stop): if self._loaded: raise OperateTimeout, timeout_message else: # raise LoadingTimeout, timeout_message self.trigger_action('Stop') #QWebPage::Stop self._loaded = True logger.warning("Page loading timeout.Force to stop the page") break gevent.sleep(2) def wait_for_alert(self): """Waits for main frame alert(). """ self.wait_for(lambda: self._alert is not None, 'User has not been alerted.') msg, self._alert = self._alert, None return msg def _release_last_resources(self): """Releases last loaded resources. :return: The released resources. """ last_resources, self.http_resources = self.http_resources[:], [] return last_resources def _page_loaded(self, success): if self.develop and self.display: if self.inspector is None: self.inspector = QWebInspector() self.inspector.setPage(self.page) self.inspector.show() scripts = [ 'atoms.js', 'htmlutils.js', ] if self.jquery_namespace: scripts.append('jquery-1.9.1.min.js', ) for script in scripts: self.evaluate_js_file(os.path.dirname(__file__) + '/javascripts/' + script) if self.jquery_namespace: self.evaluate(u"%s=jQuery.noConflict();" % self.jquery_namespace) self._loaded = True # self.cache.clear() logger.debug("Page load finished") def _page_load_started(self): logger.debug("Start load page") self._loaded = False def _unsupported_content(self, reply): """Adds an HttpResource object to http_resources with unsupported content. @param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache, reply.readAll())) def _link_clicked(self, href): """Contorl the page link clicked event,forbid open new window. @param href: The href attribute of a tag. """ self.main_frame.load(href) def _request_ended(self, reply): """Adds an HttpResource object to http_resources. @param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache)) def _authenticate(self, mix, authenticator): """Called back on basic / proxy http auth. @param mix: The QNetworkReply or QNetworkProxy object. @param authenticator: The QAuthenticator object. """ if self._auth_attempt == 0: username, password = self._auth authenticator.setUser(username) authenticator.setPassword(password) self._auth_attempt += 1 def _on_manager_ssl_errors(self, reply, errors): """Ingore all the ssl error @param reply: @param errors: """ url = unicode(reply.url().toString()) if self.ignore_ssl_errors: reply.ignoreSslErrors() else: logger.warning('SSL certificate error: %s' % url) def _exit(self): """Destroy the Qt main event loop. """ GRobot.exit_lock.acquire() if self.inspector: self.inspector.close() sip.delete(self.inspector) if self.display: self.webview.close() sip.delete(self.webview) if self.page and not sip.isdeleted(self.page): sip.delete(self.page) GRobot._liveRobot -= 1 if GRobot._liveRobot == 0 and GRobot._loop is not None: GRobot._loop.stop() GRobot._loop = None GRobot._app = None if hasattr(self, 'xvfb'): GRobot.xvfb.terminate() GRobot.exit_lock.release()
def __init__(self, user_agent=default_user_agent, operate_timeout=10, loading_timeout=60, log_level=logging.WARNING, display=False, viewport_size=(1024, 768), accept_language='en,*', ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "GRobot"), image_enabled=True, plugins_enabled=False, java_enabled=False, javascript_enabled=True, plugin_path=None, develop=False, proxy=None, sleep=0.5, jquery_namespace='GRobot'): """GRobot manages a QWebPage. @param user_agent: The default User-Agent header. @param operate_timeout: Operation timeout. @param loading_timeout: The page loading timeout. @param log_level: The optional logging level. @param display: A boolean that tells GRobot to displays UI. @param viewport_size: A tupple that sets initial viewport size. @param accept_language: Set the webkit accept language. @param ignore_ssl_errors: A boolean that forces ignore ssl errors. @param cache_dir: A directory path where to store cache datas. @param image_enabled: Enable images. @param plugins_enabled: Enable plugins (like Flash). @param java_enabled: Enable Java JRE. @param javascript_enabled: Enable Javascript. @param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins']) @param develop: Enable the Webkit Inspector. @param proxy: Set a Socks5,HTTP{S} Proxy @param sleep: Sleep `sleep` second,after operate @param jquery_namespace: Set the jQuery namespace. """ GRobot.exit_lock.acquire() logger.setLevel(log_level) plugin_path = plugin_path or ['/usr/lib/mozilla/plugins', ] GRobot._liveRobot += 1 self.develop = develop self.inspector = None self.plugin = False self.exitLoop = False self.set_proxy(proxy) self.sleep = sleep self.jquery_namespace = jquery_namespace self.popup_messages = None self.accept_language = accept_language self._loaded = True self._confirm_expected = None self._prompt_expected = None self._upload_file = None self._alert = None self.http_resources = [] self.user_agent = user_agent self.loading_timeout = loading_timeout self.operate_timeout = operate_timeout self.ignore_ssl_errors = ignore_ssl_errors if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ \ and not hasattr(GRobot, 'xvfb'): try: os.environ['DISPLAY'] = ':99' GRobot.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a GRobot run oustside ' + \ 'an X instance') self.display = display if not GRobot._app: GRobot._app = QApplication.instance() or QApplication(['GRobot']) if plugin_path: for p in plugin_path: GRobot._app.addLibraryPath(p) self.page = GRobotWebPage(self, GRobot._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() #TODO:Think about how to handle the network accessible signal #self.manager.networkAccessibleChanged.connect() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired \ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired \ .connect(self._authenticate) self.main_frame = self.page.mainFrame() self.webview = None self.viewport_size = viewport_size self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() if display else self.webview.hide() self.set_viewport_size(*viewport_size) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.page.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, self.develop) self.enable_image = image_enabled self.enable_javascript = javascript_enabled #always open link in current window instead of new window self.page.setLinkDelegationPolicy(QWebPage.DelegateAllLinks) self.page.linkClicked.connect(self._link_clicked) #start the qt main loop GRobot._loop = QtMainLoop(GRobot._app) GRobot._loop.start() GRobot.exit_lock.release()
#! /usr/bin/env python # encoding: utf-8 import sys import signal import urllib # from PyQt4.QtCore import * # from PyQt4.QtGui import * from PyQt4.QtCore import QUrl from PyQt4.QtCore import SIGNAL from PyQt4.QtGui import QApplication from PyQt4.QtNetwork import QNetworkCookieJar, QNetworkCookie from PyQt4.QtWebKit import QWebPage _cookieJar = QNetworkCookieJar() _cookie = 'a=m;\nb=1;\nc=test' def user_agent(url=""): print url return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) " \ "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36" def alert(frame, message): print message pass def console_message(s1, i, s2): pass
class Kit(object): _app = None def __init__(self, gui=False): if not Kit._app: Kit._app = QApplication([]) manager = KitNetworkAccessManager() manager.finished.connect(self.network_reply_handler) self.cookie_jar = QNetworkCookieJar() manager.setCookieJar(self.cookie_jar) self.page = KitPage() self.page.setNetworkAccessManager(manager) self.view = KitWebView() self.view.setPage(self.page) self.view.setApplication(Kit._app) if gui: self.view.show() def get_cookies(self): cookies = {} for cookie in self.cookie_jar.allCookies(): cookies[cookie.name().data()] = cookie.value().data() return cookies def request(self, url, user_agent='Mozilla', cookies=None, timeout=15, method='get', data=None, headers=None): if cookies is None: cookies = {} if headers is None: headers = {} url_info = urlsplit(url) self.resource_list = [] loop = QEventLoop() self.view.loadFinished.connect(loop.quit) # Timeout timer = QTimer() timer.setSingleShot(True) timer.timeout.connect(loop.quit) timer.start(timeout * 1000) # User-Agent self.page.user_agent = user_agent # Cookies cookie_obj_list = [] for name, value in cookies.items(): domain = ('.' + url_info.netloc).split(':')[0] #print 'CREATE COOKIE %s=%s' % (name, value) #print 'DOMAIN = %s' % domain cookie_obj = QNetworkCookie(name, value) cookie_obj.setDomain(domain) cookie_obj_list.append(cookie_obj) self.cookie_jar.setAllCookies(cookie_obj_list) # Method method_obj = getattr(QNetworkAccessManager, '%sOperation' % method.capitalize()) # Ensure that Content-Type is correct if method is post if method == 'post': headers['Content-Type'] = 'application/x-www-form-urlencoded' # Post data if data is None: data = QByteArray() # Request object request_obj = QNetworkRequest(QUrl(url)) # Headers for name, value in headers.items(): request_obj.setRawHeader(name, value) # Make a request self.view.load(request_obj, method_obj, data) loop.exec_() if timer.isActive(): request_resource = None url = str(self.page.mainFrame().url().toString()).rstrip('/') for res in self.resource_list: if url == res.url or url == res.url.rstrip('/'): request_resource = res break if request_resource: return self.build_response(request_resource) else: raise KitError('Request was successful but it is not possible' ' to associate the request to one of received' ' responses') else: raise KitError('Timeout while loading %s' % url) def build_response(self, resource): response = Response() response.head = '' response.code = resource.status_code runtime_body = self.page.mainFrame().toHtml() body = resource.reply.data url = resource.reply.url().toString() headers = resource.headers cookies = self.get_cookies() # py3 hack if PY3K: if isinstance(body, QByteArray): body = body.data() headers = decode_dict(headers) cookies = decode_dict(cookies) else: runtime_body = unicode(runtime_body) body = str(body) url = str(url) response.runtime_body = runtime_body.encode('utf-8') response.body = body response.url = url response.parse(charset='utf-8') response.headers = headers response.cookies = cookies return response def __del__(self): self.view.setPage(None) def network_reply_handler(self, reply): status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if status_code: if not isinstance(status_code, int): status_code = status_code.toInt()[0] logger.debug('Resource loaded: %s [%d]' % (reply.url().toString(), status_code)) self.resource_list.append(Resource(reply))
def initProperty(): AppProperty.AppTitle = "QPA" AppProperty.AppIcon = QIcon("imgs/icon.png") AppProperty.HomeDir = getcwd() AppProperty.CookieJar = QNetworkCookieJar()
class Ghost(object): """Ghost manages a QWebPage. :param user_agent: The default User-Agent header. :param wait_timeout: Maximum step duration in second. :param wait_callback: An optional callable that is periodically executed until Ghost stops waiting. :param log_level: The optional logging level. :param display: A boolean that tells ghost to displays UI. :param viewport_size: A tupple that sets initial viewport size. """ _alert = None _confirm_expected = None _prompt_expected = None _upload_file = None _app = None def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), cache_dir='/tmp/ghost.py'): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) self.page = GhostWebPage(Ghost._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) self.page.setForwardUnsupportedContent(True) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() def __del__(self): self.exit() def capture(self, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Returns snapshot as QImage. :param region: An optional tupple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ if region is None and selector is not None: region = self.region_for_selector(selector) if region: x1, y1, x2, y2 = region w, h = (x2 - x1), (y2 - y1) image = QImage(QSize(x2, y2), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() image = image.copy(x1, y1, w, h) else: image = QImage(self.page.viewportSize(), format) painter = QPainter(image) self.main_frame.render(painter) painter.end() return image def capture_to(self, path, region=None, selector=None, format=QImage.Format_ARGB32_Premultiplied): """Saves snapshot as image. :param path: The destination path. :param region: An optional tupple containing region as pixel coodinates. :param selector: A selector targeted the element to crop on. :param format: The output image format. """ self.capture(region=region, format=format, selector=selector).save(path) @client_utils_required @can_load_page def click(self, selector): """Click the targeted element. :param selector: A CSS3 selector to targeted element. """ if not self.exists(selector): raise Exception("Can't find element to click") return self.evaluate('GhostUtils.click("%s");' % selector) class confirm: """Statement that tells Ghost how to deal with javascript confirm(). :param confirm: A bollean that confirm. :param callable: A callable that returns a boolean for confirmation. """ def __init__(self, confirm=True, callback=None): self.confirm = confirm self.callback = callback def __enter__(self): Ghost._confirm_expected = (self.confirm, self.callback) def __exit__(self, type, value, traceback): Ghost._confirm_expected = None @property def content(self): """Returns current frame HTML as a string.""" return unicode(self.main_frame.toHtml()) @property def cookies(self): """Returns all cookies.""" return self.cookie_jar.allCookies() def delete_cookies(self): """Deletes all cookies.""" self.cookie_jar.setAllCookies([]) @can_load_page def evaluate(self, script): """Evaluates script in page frame. :param script: The script to evaluate. """ return (self.main_frame.evaluateJavaScript("%s" % script), self._release_last_resources()) def evaluate_js_file(self, path, encoding='utf-8'): """Evaluates javascript file at given path in current frame. Raises native IOException in case of invalid file. :param path: The path of the file. :param encoding: The file's encoding. """ self.evaluate(codecs.open(path, encoding=encoding).read()) def exists(self, selector): """Checks if element exists for given selector. :param string: The element selector. """ return not self.main_frame.findFirstElement(selector).isNull() def exit(self): """Exits application and relateds.""" if self.display: self.webview.close() Ghost._app.quit() del self.manager del self.page del self.main_frame if hasattr(self, 'xvfb'): self.xvfb.terminate() @can_load_page def fill(self, selector, values): """Fills a form with provided values. :param selector: A CSS selector to the target form to fill. :param values: A dict containing the values. """ if not self.exists(selector): raise Exception("Can't find form") resources = [] for field in values: r, res = self.set_field_value("%s [name=%s]" % (selector, field), values[field]) resources.extend(res) return True, resources @client_utils_required @can_load_page def fire_on(self, selector, method): """Call method on element matching given selector. :param selector: A CSS selector to the target element. :param method: The name of the method to fire. :param expect_loading: Specifies if a page loading is expected. """ return self.evaluate('GhostUtils.fireOn("%s", "%s");' % (selector, method)) def global_exists(self, global_name): """Checks if javascript global exists. :param global_name: The name of the global. """ return self.evaluate('!(typeof %s === "undefined");' % global_name)[0] def hide(self): """Close the webview.""" try: self.webview.close() except: raise Exception("no webview to close") def open(self, address, method='get', headers={}, auth=None): """Opens a web page. :param address: The resource URL. :param method: The Http method. :param headers: An optional dict of extra request hearders. :param auth: An optional tupple of HTTP auth (username, password). :return: Page resource, All loaded resources. """ body = QByteArray() try: method = getattr(QNetworkAccessManager, "%sOperation" % method.capitalize()) except AttributeError: raise Exception("Invalid http method %s" % method) request = QNetworkRequest(QUrl(address)) request.CacheLoadControl(0) for header in headers: request.setRawHeader(header, headers[header]) self._auth = auth self._auth_attempt = 0 # Avoids reccursion self.main_frame.load(request, method, body) self.loaded = False return self.wait_for_page_loaded() class prompt: """Statement that tells Ghost how to deal with javascript prompt(). :param value: A string value to fill in prompt. :param callback: A callable that returns the value to fill in. """ def __init__(self, value='', callback=None): self.value = value self.callback = callback def __enter__(self): Ghost._prompt_expected = (self.value, self.callback) def __exit__(self, type, value, traceback): Ghost._prompt_expected = None @client_utils_required def region_for_selector(self, selector): """Returns frame region for given selector as tupple. :param selector: The targeted element. """ geo = self.main_frame.findFirstElement(selector).geometry() try: region = (geo.left(), geo.top(), geo.right(), geo.bottom()) except: raise Exception("can't get region for selector '%s'" % selector) return region @can_load_page @client_utils_required def set_field_value(self, selector, value, blur=True): """Sets the value of the field matched by given selector. :param selector: A CSS selector that target the field. :param value: The value to fill in. :param blur: An optional boolean that force blur when filled in. """ def _set_text_value(selector, value): return self.evaluate('document.querySelector("%s").value=%s;' % (selector, json.dumps(value))) res, resources = None, [] element = self.main_frame.findFirstElement(selector) if element.isNull(): raise Exception('can\'t find element for %s"' % selector) self.fire_on(selector, 'focus') if element.tagName() in ["TEXTAREA", "SELECT"]: res, resources = _set_text_value(selector, value) elif element.tagName() == "INPUT": if element.attribute('type') in [ "color", "date", "datetime", "datetime-local", "email", "hidden", "month", "number", "password", "range", "search", "tel", "text", "time", "url", "week" ]: res, resources = _set_text_value(selector, value) elif element.attribute('type') == "checkbox": res, resources = self.evaluate( 'GhostUtils.setCheckboxValue("%s", %s);' % (selector, json.dumps(value))) elif element.attribute('type') == "radio": res, resources = self.evaluate( 'GhostUtils.setRadioValue("%s", %s);' % (selector, json.dumps(value))) elif element.attribute('type') == "file": Ghost._upload_file = value res, resources = self.click(selector) Ghost._upload_file = None else: raise Exception('unsuported field tag') if blur: self.fire_on(selector, 'blur') return res, resources def set_viewport_size(self, width, height): """Sets the page viewport size. :param width: An integer that sets width pixel count. :param height: An integer that sets height pixel count. """ self.page.setViewportSize(QSize(width, height)) def show(self): """Show current page inside a QWebView. """ self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show() def wait_for(self, condition, timeout_message): """Waits until condition is True. :param condition: A callable that returns the condition. :param timeout_message: The exception message on timeout. """ started_at = time.time() while not condition(): if time.time() > (started_at + self.wait_timeout): raise Exception(timeout_message) time.sleep(0.01) Ghost._app.processEvents() if self.wait_callback is not None: self.wait_callback() def wait_for_alert(self): """Waits for main frame alert(). """ self.wait_for(lambda: Ghost._alert is not None, 'User has not been alerted.') msg = Ghost._alert Ghost._alert = None return msg, self._release_last_resources() def wait_for_page_loaded(self): """Waits until page is loaded, assumed that a page as been requested. """ self.wait_for(lambda: self.loaded, 'Unable to load requested page') resources = self._release_last_resources() page = None url = self.main_frame.url().toString() for resource in resources: if url == resource.url: page = resource return page, resources def wait_for_selector(self, selector): """Waits until selector match an element on the frame. :param selector: The selector to wait for. """ self.wait_for(lambda: self.exists(selector), 'Can\'t find element matching "%s"' % selector) return True, self._release_last_resources() def wait_for_text(self, text): """Waits until given text appear on main frame. :param text: The text to wait for. """ self.wait_for(lambda: text in self.content, 'Can\'t find "%s" in current frame' % text) return True, self._release_last_resources() def _authenticate(self, mix, authenticator): """Called back on basic / proxy http auth. :param mix: The QNetworkReply or QNetworkProxy object. :param authenticator: The QAuthenticator object. """ if self._auth_attempt == 0: username, password = self._auth authenticator.setUser(username) authenticator.setPassword(password) self._auth_attempt += 1 def _page_loaded(self): """Called back when page is loaded. """ self.loaded = True self.cache.clear() def _page_load_started(self): """Called back when page load started. """ self.loaded = False def _release_last_resources(self): """Releases last loaded resources. :return: The released resources. """ last_resources = self.http_resources self.http_resources = [] return last_resources def _request_ended(self, reply): """Adds an HttpResource object to http_resources. :param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append(HttpResource(reply, self.cache)) def _unsupported_content(self, reply): """Adds an HttpResource object to http_resources with unsupported content. :param reply: The QNetworkReply object. """ if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute): self.http_resources.append( HttpResource(reply, self.cache, reply.readAll()))
def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), plugins_enabled=False, java_enabled=False, plugin_path=['/usr/lib/mozilla/plugins',], download_images=True, qt_debug=False): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Error('Xvfb is required to a ghost run outside ' + 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) if plugin_path: for p in plugin_path: Ghost._app.addLibraryPath(p) self.popup_messages = [] self.page = GhostWebPage(Ghost._app, self) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) self.page.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, download_images) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: class MyQWebView(QtWebKit.QWebView): def sizeHint(self): return QSize(*viewport_size) self.webview = MyQWebView() if plugins_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True) if java_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, True) self.webview.setPage(self.page) self.webview.show() else: self.webview = None
def setCookiesFromUrl(self, cookieList, url): """ Public method to set cookies for a URL. @param cookieList list of cookies to set (list of QNetworkCookie) @param url url to set cookies for (QUrl) @return flag indicating cookies were set (boolean) """ if not self.__loaded: self.load() globalSettings = QWebSettings.globalSettings() if globalSettings.testAttribute(QWebSettings.PrivateBrowsingEnabled): return False host = url.host() eBlock = self.__isOnDomainList(self.__exceptionsBlock, host) eAllow = not eBlock and \ self.__isOnDomainList(self.__exceptionsAllow, host) eAllowSession = not eBlock and \ not eAllow and \ self.__isOnDomainList(self.__exceptionsAllowForSession, host) addedCookies = False acceptInitially = self.__acceptCookies != self.AcceptNever if (acceptInitially and not eBlock) or \ (not acceptInitially and (eAllow or eAllowSession)): # url domain == cookie domain soon = QDateTime.currentDateTime() soon = soon.addDays(90) for cookie in cookieList: lst = [] if not (self.__filterTrackingCookies and \ cookie.name().startsWith("__utm")): if eAllowSession: cookie.setExpirationDate(QDateTime()) if self.__keepCookies == self.KeepUntilTimeLimit and \ not cookie.isSessionCookie and \ cookie.expirationDate() > soon: cookie.setExpirationDate(soon) lst.append(cookie) if QNetworkCookieJar.setCookiesFromUrl(self, lst, url): addedCookies = True elif self.__acceptCookies == self.AcceptAlways: # force it in if wanted cookies = self.allCookies() for ocookie in cookies[:]: # does the cookie exist already? if cookie.name() == ocookie.name() and \ cookie.domain() == ocookie.domain() and \ cookie.path() == ocookie.path(): # found a match cookies.remove(ocookie) cookies.append(cookie) self.setAllCookies(cookies) addedCookies = True if addedCookies: self.__saveTimer.changeOccurred() self.emit(SIGNAL("cookiesChanged()")) return addedCookies
def __init__(self, framework, parent = None): QNetworkCookieJar.__init__(self, parent) self.framework = framework self.__read_only = False
class Kit(object): _app = None def __init__(self, gui=False): if not Kit._app: Kit._app = QApplication([]) manager = KitNetworkAccessManager() manager.finished.connect(self.network_reply_handler) self.cookie_jar = QNetworkCookieJar() manager.setCookieJar(self.cookie_jar) self.page = KitPage() self.page.setNetworkAccessManager(manager) self.view = KitWebView() self.view.setPage(self.page) self.view.setApplication(Kit._app) if gui: self.view.show() def get_cookies(self): cookies = {} for cookie in self.cookie_jar.allCookies(): cookies[cookie.name().data()] = cookie.value().data() return cookies def request(self, url, user_agent='Mozilla', cookies={}, timeout=15, method='get', data=None, headers={}): url_info = urlsplit(url) self.resource_list = [] loop = QEventLoop() self.view.loadFinished.connect(loop.quit) # Timeout timer = QTimer() timer.setSingleShot(True) timer.timeout.connect(loop.quit) timer.start(timeout * 1000) # User-Agent self.page.user_agent = user_agent # Cookies cookie_obj_list = [] for name, value in cookies.items(): domain = ('.' + url_info.netloc).split(':')[0] #print 'CREATE COOKIE %s=%s' % (name, value) #print 'DOMAIN = %s' % domain cookie_obj = QNetworkCookie(name, value) cookie_obj.setDomain(domain) cookie_obj_list.append(cookie_obj) self.cookie_jar.setAllCookies(cookie_obj_list) # Method method_obj = getattr(QNetworkAccessManager, '%sOperation' % method.capitalize()) # Ensure that Content-Type is correct if method is post if method == 'post': headers['Content-Type'] = 'application/x-www-form-urlencoded' # Post data if data is None: data = QByteArray() # Request object request_obj = QNetworkRequest(QUrl(url)) # Headers for name, value in headers.items(): request_obj.setRawHeader(name, value) # Make a request self.view.load(request_obj, method_obj, data) loop.exec_() if timer.isActive(): request_resource = None url = str(self.page.mainFrame().url().toString()).rstrip('/') for res in self.resource_list: if url == res.url or url == res.url.rstrip('/'): request_resource = res break if request_resource: return self.build_response(request_resource) else: raise KitError('Request was successfull but it is not possible '\ 'to associate the request to one of received responses') else: raise KitError('Timeout while loading %s' % url) def build_response(self, resource): response = Response() response.head = '' response.code = resource.status_code runtime_body = self.page.mainFrame().toHtml() body = resource.reply.data url = resource.reply.url().toString() headers = resource.headers cookies = self.get_cookies() # py3 hack if PY3K: if isinstance(body, QByteArray): body = body.data() headers = decode_dict(headers) cookies = decode_dict(cookies) else: runtime_body = unicode(runtime_body) body = str(body) url = str(url) response.runtime_body = runtime_body.encode('utf-8') response.body = body response.url = url response.parse(charset='utf-8') response.headers = headers response.cookies = cookies return response def __del__(self): self.view.setPage(None) def network_reply_handler(self, reply): status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if status_code: if not isinstance(status_code, int): status_code = status_code.toInt()[0] logger.debug('Resource loaded: %s [%d]' % (reply.url().toString(), status_code)) self.resource_list.append(Resource(reply))
def cookiesForUrl(self, url): return QNetworkCookieJar.cookiesForUrl(self, url)
def __init__( self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), ): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith("win") and not "DISPLAY" in os.environ and not hasattr(Ghost, "xvfb"): try: os.environ["DISPLAY"] = ":99" Ghost.xvfb = subprocess.Popen(["Xvfb", ":99"]) except OSError: raise Exception("Xvfb is required to a ghost run oustside " + "an X instance") self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(["ghost"]) self.page = GhostWebPage(Ghost._app) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) self.page.setForwardUnsupportedContent(True) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired.connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired.connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: self.webview = QtWebKit.QWebView() self.webview.setPage(self.page) self.webview.show()