class HtmlRender(object):
def __init__(self, url, baseurl=None):
self.url = url
self.web_view = QWebView()
self.network_manager = SplashQNetworkAccessManager()
self.web_page = SplashQWebPage()
self.web_page.setNetworkAccessManager(self.network_manager)
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
settings = self.web_view.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.deferred = defer.Deferred()
request = QNetworkRequest()
request.setUrl(QUrl(url))
if baseurl:
self._baseUrl = QUrl(baseurl)
self.network_manager.finished.connect(self._requestFinished)
self.network_manager.get(request)
else:
self.web_page.loadFinished.connect(self._loadFinished)
self.web_page.mainFrame().load(request)
def close(self):
self.web_view.stop()
self.web_view.close()
self.web_page.deleteLater()
self.web_view.deleteLater()
self.network_manager.deleteLater()
def _requestFinished(self, reply):
self.web_page.networkAccessManager().finished.disconnect(self._requestFinished)
self.web_view.loadFinished.connect(self._loadFinished)
mimeType = reply.header(QNetworkRequest.ContentTypeHeader).toString()
self.web_view.page().mainFrame().setContent(reply.readAll(), mimeType, self._baseUrl)
def _loadFinished(self, ok):
if self.deferred.called:
return
if ok:
try:
self.deferred.callback(self._render())
except:
self.deferred.errback()
else:
self.deferred.errback(RenderError())
def _render(self):
frame = self.web_view.page().mainFrame()
return str(frame.toHtml().toUtf8())
class WebpageRender(object):
def __init__(self, network_manager, splash_proxy_factory, splash_request, verbose=False):
self.network_manager = network_manager
self.web_view = QWebView()
self.web_page = SplashQWebPage()
self.web_page.setNetworkAccessManager(self.network_manager)
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
settings = self.web_page.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls, True)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.splash_request = splash_request
self.web_page.splash_request = splash_request
self.web_page.splash_proxy_factory = splash_proxy_factory
self.verbose = verbose
self.deferred = defer.Deferred()
# ======= General request/response handling:
def doRequest(
self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, console=False
):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
request = QNetworkRequest()
request.setUrl(QUrl(url.decode("utf8")))
if self.viewport != "full":
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if getattr(self.splash_request, "pass_headers", False):
headers = self.splash_request.getAllHeaders()
for name, value in headers.items():
request.setRawHeader(name, value)
if name.lower() == "user-agent":
self.web_page.custom_user_agent = value
if baseurl:
self._baseUrl = QUrl(baseurl.decode("utf8"))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == "POST":
self.web_page.mainFrame().load(
request, QNetworkAccessManager.PostOperation, self.splash_request.content.getvalue()
)
else:
self.web_page.mainFrame().load(request)
def close(self):
self.web_view.stop()
self.web_view.close()
self.web_page.deleteLater()
self.web_view.deleteLater()
def _requestFinished(self):
self.log("_requestFinished %s" % id(self.splash_request))
self.web_page.loadFinished.connect(self._loadFinished)
mimeType = self._reply.header(QNetworkRequest.ContentTypeHeader).toString()
data = self._reply.readAll()
self.web_page.mainFrame().setContent(data, mimeType, self._baseUrl)
if self._reply.error():
log.msg("Error loading %s: %s" % (self.url, self._reply.errorString()), system="render")
self._reply.close()
self._reply.deleteLater()
def _loadFinished(self, ok):
self.log("_loadFinished %s" % id(self.splash_request))
if self.deferred.called:
# sometimes this callback is called multiple times
self.log("_loadFinished called multiple times")
return
page_ok = ok and self.web_page.errorInfo is None
maybe_redirect = not ok and self.web_page.errorInfo is None
error_loading = ok and self.web_page.errorInfo is not None
if maybe_redirect:
self.log("Redirect or other non-fatal error detected %s" % id(self.splash_request))
# XXX: It assumes loadFinished will be called again because
# redirect happens. If redirect is detected improperly,
# loadFinished won't be called again, and Splash will return
# the result only after a timeout.
#
# FIXME: This can happen if server returned incorrect
# Content-Type header; there is no an additional loadFinished
# signal in this case.
return
if page_ok:
time_ms = int(self.wait_time * 1000)
QTimer.singleShot(time_ms, self._loadFinishedOK)
elif error_loading:
self.log("loadFinished %s: %s" % (id(self.splash_request), str(self.web_page.errorInfo))) # , min_level=1)
# XXX: maybe return a meaningful error page instead of generic
# error message?
self.deferred.errback(RenderError())
else:
self.log("loadFinished %s: unknown error" % id(self.splash_request)) # , min_level=1)
self.deferred.errback(RenderError())
def _loadFinishedOK(self):
self.log("_loadFinishedOK %s" % id(self.splash_request))
try:
self._prerender()
self.deferred.callback(self._render())
except:
self.deferred.errback()
# ======= Rendering methods that subclasses can use:
def _getHtml(self):
frame = self.web_page.mainFrame()
return bytes(frame.toHtml().toUtf8())
def _getPng(self, width=None, height=None):
image = QImage(self.web_page.viewportSize(), QImage.Format_ARGB32)
painter = QPainter(image)
self.web_page.mainFrame().render(painter)
painter.end()
if width:
image = image.scaledToWidth(width, Qt.SmoothTransformation)
if height:
image = image.copy(0, 0, width, height)
b = QBuffer()
image.save(b, "png")
return bytes(b.data())
def _getIframes(self, children=True, html=True):
frame = self.web_page.mainFrame()
return self._frameToDict(frame, children, html)
def _render(self):
raise NotImplementedError()
# ======= Other helper methods:
def _setViewportSize(self, viewport):
w, h = map(int, viewport.split("x"))
size = QSize(w, h)
self.web_page.setViewportSize(size)
def _setFullViewport(self):
size = self.web_page.mainFrame().contentsSize()
if size.isEmpty():
self.log("contentsSize method doesn't work %s" % id(self.splash_request))
self._setViewportSize(defaults.VIEWPORT_FALLBACK)
else:
self.web_page.setViewportSize(size)
def _loadJsLibs(self, frame, js_profile):
if js_profile:
for jsfile in os.listdir(js_profile):
if jsfile.endswith(".js"):
with open(os.path.join(js_profile, jsfile)) as f:
frame.evaluateJavaScript(f.read().decode("utf-8"))
def _runJS(self, js_source, js_profile):
js_output = None
js_console_output = None
if js_source:
frame = self.web_page.mainFrame()
if self.console:
js_console = JavascriptConsole()
frame.addToJavaScriptWindowObject("console", js_console)
if js_profile:
self._loadJsLibs(frame, js_profile)
ret = frame.evaluateJavaScript(js_source)
js_output = bytes(ret.toString().toUtf8())
if self.console:
js_console_output = [bytes(s.toUtf8()) for s in js_console.messages]
return js_output, js_console_output
def _frameToDict(self, frame, children=True, html=True):
g = frame.geometry()
res = {
"url": unicode(frame.url().toString()),
"requestedUrl": unicode(frame.requestedUrl().toString()),
"geometry": (g.x(), g.y(), g.width(), g.height()),
"title": unicode(frame.title()),
}
if html:
res["html"] = unicode(frame.toHtml())
if children:
res["childFrames"] = [self._frameToDict(f, True, html) for f in frame.childFrames()]
res["frameName"] = unicode(frame.frameName())
return res
def _prerender(self):
if self.viewport == "full":
self._setFullViewport()
self.js_output, self.js_console_output = self._runJS(self.js_source, self.js_profile)
def log(self, text):
if self.verbose:
log.msg(text, system="render")
class BrowserTab(QObject):
"""
An object for controlling a single browser tab (QWebView).
It is created by splash.pool.Pool. Pool attaches to tab's deferred
and waits until either a callback or an errback is called, then destroys
a BrowserTab.
XXX: currently cookies are not shared between "browser tabs".
"""
def __init__(self, network_manager, splash_proxy_factory, verbosity,
render_options):
""" Create a new browser tab. """
QObject.__init__(self)
self.deferred = defer.Deferred()
self.network_manager = network_manager
self.verbosity = verbosity
self._uid = render_options.get_uid()
self._closing = False
self._active_timers = set()
self._timers_to_cancel_on_redirect = weakref.WeakKeyDictionary() # timer: callback
self._timers_to_cancel_on_error = weakref.WeakKeyDictionary() # timer: callback
self._js_console = None
self._history = []
self._autoload_scripts = []
self._init_webpage(verbosity, network_manager, splash_proxy_factory,
render_options)
self._setup_logging(verbosity)
self.http_client = _SplashHttpClient(self.web_page)
def _init_webpage(self, verbosity, network_manager, splash_proxy_factory, render_options):
""" Create and initialize QWebPage and QWebView """
self.web_page = SplashQWebPage(verbosity)
self.web_page.setNetworkAccessManager(network_manager)
self.web_page.splash_proxy_factory = splash_proxy_factory
self.web_page.render_options = render_options
self._set_default_webpage_options(self.web_page)
self._setup_webpage_events()
self.web_view = QWebView()
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
def _set_default_webpage_options(self, web_page):
"""
Set QWebPage options.
TODO: allow to customize them.
"""
settings = web_page.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls, True)
web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
def _setup_logging(self, verbosity):
""" Setup logging of various events """
self.logger = _BrowserTabLogger(
uid=self._uid,
web_page=self.web_page,
verbosity=verbosity,
)
self.logger.enable()
def _setup_webpage_events(self):
self._load_finished = WrappedSignal(self.web_page.mainFrame().loadFinished)
self.web_page.mainFrame().loadFinished.connect(self._on_load_finished)
self.web_page.mainFrame().urlChanged.connect(self._on_url_changed)
self.web_page.mainFrame().javaScriptWindowObjectCleared.connect(self._on_javascript_window_object_cleared)
def return_result(self, result):
""" Return a result to the Pool. """
if self._result_already_returned():
self.logger.log("error: result is already returned", min_level=1)
self.deferred.callback(result)
# self.deferred = None
def return_error(self, error=None):
""" Return an error to the Pool. """
if self._result_already_returned():
self.logger.log("error: result is already returned", min_level=1)
self.deferred.errback(error)
# self.deferred = None
def _result_already_returned(self):
""" Return True if an error or a result is already returned to Pool """
return self.deferred.called
def set_custom_headers(self, headers):
"""
Set custom HTTP headers to be sent with each request. Passed headers
are merged with QWebKit default headers, overwriting QWebKit values
in case of conflicts.
"""
self.web_page.custom_headers = headers
def set_images_enabled(self, enabled):
self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, enabled)
def set_viewport(self, size):
"""
Set viewport size.
If size is "full" viewport size is detected automatically.
If can also be "<width>x<height>".
"""
if size == 'full':
size = self.web_page.mainFrame().contentsSize()
if size.isEmpty():
self.logger.log("contentsSize method doesn't work %s", min_level=1)
size = defaults.VIEWPORT_FALLBACK
if not isinstance(size, QSize):
w, h = map(int, size.split('x'))
size = QSize(w, h)
self.web_page.setViewportSize(size)
w, h = int(size.width()), int(size.height())
self.logger.log("viewport size is set to %sx%s" % (w, h), min_level=2)
return w, h
def lock_navigation(self):
self.web_page.navigation_locked = True
def unlock_navigation(self):
self.web_page.navigation_locked = False
def set_content(self, data, callback, errback, mime_type=None, baseurl=None):
"""
Set page contents to ``data``, then wait until page loads.
Invoke a callback if load was successful or errback if it wasn't.
"""
if mime_type is None:
mime_type = "text/html; charset=utf-8"
if baseurl is None:
baseurl = ''
if isinstance(data, unicode):
data = data.encode('utf8')
callback_id = self._load_finished.connect(
self._on_content_ready,
callback=callback,
errback=errback,
)
self.logger.log("callback %s is connected to loadFinished" % callback_id, min_level=3)
self.web_page.mainFrame().setContent(data, mime_type, QUrl(baseurl))
def set_user_agent(self, value):
""" Set User-Agent header for future requests """
self.http_client.set_user_agent(value)
def get_cookies(self):
""" Return a list of all cookies in the current cookiejar """
return cookies2har(self.web_page.cookiejar.allCookies())
def init_cookies(self, cookies):
""" Replace all current cookies with ``cookies`` """
self.web_page.cookiejar.init(cookies)
def clear_cookies(self):
""" Remove all cookies. Return a number of cookies deleted. """
return self.web_page.cookiejar.clear()
def delete_cookies(self, name=None, url=None):
"""
Delete cookies with name == ``name``.
If ``url`` is not None then only those cookies are deleted wihch
are to be added when a request is sent to ``url``.
Return a number of cookies deleted.
"""
return self.web_page.cookiejar.delete(name, url)
def add_cookie(self, cookie):
return self.web_page.cookiejar.add(cookie)
@property
def url(self):
""" Current URL """
return unicode(self.web_page.mainFrame().url().toString())
def go(self, url, callback, errback, baseurl=None, http_method='GET',
body=None, headers=None):
"""
Go to an URL. This is similar to entering an URL in
address tab and pressing Enter.
"""
self.store_har_timing("_onStarted")
if baseurl:
# If baseurl is used, we download the page manually,
# then set its contents to the QWebPage and let it
# download related resources and render the result.
cb = functools.partial(
self._on_baseurl_request_finished,
callback=callback,
errback=errback,
baseurl=baseurl,
url=url,
)
self.http_client.request(url,
callback=cb,
method=http_method,
body=body,
headers=headers,
follow_redirects=True,
)
else:
# if not self._goto_callbacks.isempty():
# self.logger.log("Only a single concurrent 'go' request is supported. "
# "Previous go requests will be cancelled.", min_level=1)
# # When a new URL is loaded to mainFrame an errback will
# # be called, so we're not cancelling this callback manually.
callback_id = self._load_finished.connect(
self._on_content_ready,
callback=callback,
errback=errback,
)
self.logger.log("callback %s is connected to loadFinished" % callback_id, min_level=3)
self._load_url_to_mainframe(url, http_method, body, headers=headers)
def stop_loading(self):
"""
Stop loading of the current page and all pending page
refresh/redirect requests.
"""
self.logger.log("stop_loading", min_level=2)
self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)
self.web_view.stop()
def close(self):
""" Destroy this tab """
self._closing = True
self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)
self.web_view.stop()
self.web_view.close()
self.web_page.deleteLater()
self.web_view.deleteLater()
self._cancel_all_timers()
@skip_if_closing
def _on_load_finished(self, ok):
if self.web_page.maybe_redirect(ok):
self.logger.log("Redirect or other non-fatal error detected", min_level=2)
return
if self.web_page.is_ok(ok): # or maybe_redirect:
self.logger.log("loadFinished: ok", min_level=2)
else:
self._cancel_timers(self._timers_to_cancel_on_error)
if self.web_page.error_loading(ok):
self.logger.log("loadFinished: %s" % (str(self.web_page.error_info)), min_level=1)
else:
self.logger.log("loadFinished: unknown error", min_level=1)
def _on_baseurl_request_finished(self, callback, errback, baseurl, url):
"""
This method is called when ``baseurl`` is used and a
reply for the first request is received.
"""
self.logger.log("baseurl_request_finished", min_level=2)
reply = self.sender()
mime_type = reply.header(QNetworkRequest.ContentTypeHeader).toString()
data = reply.readAll()
self.set_content(
data=data,
callback=callback,
errback=errback,
mime_type=mime_type,
baseurl=baseurl,
)
if reply.error():
self.logger.log("Error loading %s: %s" % (url, reply.errorString()), min_level=1)
def _load_url_to_mainframe(self, url, http_method, body=None, headers=None):
request = self.http_client.request_obj(url, headers=headers)
meth = OPERATION_QT_CONSTANTS[http_method]
if body is None: # PyQT doesn't support body=None
self.web_page.mainFrame().load(request, meth)
else:
self.web_page.mainFrame().load(request, meth, body)
@skip_if_closing
def _on_content_ready(self, ok, callback, errback, callback_id):
"""
This method is called when a QWebPage finishes loading its contents.
"""
if self.web_page.maybe_redirect(ok):
# XXX: It assumes loadFinished will be called again because
# redirect happens. If redirect is detected improperly,
# loadFinished won't be called again, and Splash will return
# the result only after a timeout.
return
self.logger.log("loadFinished: disconnecting callback %s" % callback_id, min_level=3)
self._load_finished.disconnect(callback_id)
if self.web_page.is_ok(ok):
callback()
elif self.web_page.error_loading(ok):
# XXX: maybe return a meaningful error page instead of generic
# error message?
errback()
# errback(RenderError())
else:
errback()
# errback(RenderError())
def wait(self, time_ms, callback, onredirect=None, onerror=None):
"""
Wait for time_ms, then run callback.
If onredirect is True then the timer is cancelled if redirect happens.
If onredirect is callable then in case of redirect the timer is
cancelled and this callable is called.
If onerror is True then the timer is cancelled if a render error
happens. If onerror is callable then in case of a render error the
timer is cancelled and this callable is called.
"""
timer = QTimer()
timer.setSingleShot(True)
timer_callback = functools.partial(self._on_wait_timeout,
timer=timer,
callback=callback,
)
timer.timeout.connect(timer_callback)
self.logger.log("waiting %sms; timer %s" % (time_ms, id(timer)), min_level=2)
timer.start(time_ms)
self._active_timers.add(timer)
if onredirect:
self._timers_to_cancel_on_redirect[timer] = onredirect
if onerror:
self._timers_to_cancel_on_error[timer] = onerror
def _on_wait_timeout(self, timer, callback):
self.logger.log("wait timeout for %s" % id(timer), min_level=2)
if timer in self._active_timers:
self._active_timers.remove(timer)
self._timers_to_cancel_on_redirect.pop(timer, None)
self._timers_to_cancel_on_error.pop(timer, None)
callback()
def _cancel_timer(self, timer, errback=None):
self.logger.log("cancelling timer %s" % id(timer), min_level=2)
if timer in self._active_timers:
self._active_timers.remove(timer)
timer.stop()
try:
if callable(errback):
self.logger.log("calling timer errback", min_level=2)
errback()
finally:
timer.deleteLater()
def _cancel_timers(self, timers):
for timer, oncancel in list(timers.items()):
self._cancel_timer(timer, oncancel)
timers.pop(timer, None)
def _cancel_all_timers(self):
self.logger.log("cancelling %d remaining timers" % len(self._active_timers), min_level=2)
for timer in list(self._active_timers):
self._cancel_timer(timer)
def _on_url_changed(self, url):
# log history
url = unicode(url.toString())
cause_ev = self.web_page.har_log._prev_entry(url, -1)
if cause_ev:
self._history.append(without_private(cause_ev.data))
self._cancel_timers(self._timers_to_cancel_on_redirect)
def run_js_file(self, filename):
"""
Load JS library from file ``filename`` to the current frame.
"""
with open(filename, 'rb') as f:
script = f.read().decode('utf-8')
return self.runjs(script)
def run_js_files(self, folder):
"""
Load all JS libraries from ``folder`` folder to the current frame.
"""
for jsfile in os.listdir(folder):
if jsfile.endswith('.js'):
filename = os.path.join(folder, jsfile)
self.run_js_file(filename)
def autoload(self, js_source):
""" Execute JS code before each page load """
self._autoload_scripts.append(js_source)
def no_autoload(self):
""" Remove all scripts scheduled for auto-loading """
self._autoload_scripts = []
def _on_javascript_window_object_cleared(self):
for script in self._autoload_scripts:
self.web_page.mainFrame().evaluateJavaScript(script)
def http_get(self, url, callback, headers=None, follow_redirects=True):
""" Send a GET request; call a callback with the reply as an argument. """
self.http_client.get(url,
callback=callback,
headers=headers,
follow_redirects=follow_redirects
)
def runjs(self, js_source):
"""
Run JS code in page context and return the result.
Only string results are supported.
"""
frame = self.web_page.mainFrame()
res = frame.evaluateJavaScript(js_source)
return qt2py(res)
def store_har_timing(self, name):
self.web_page.har_log.store_timing(name)
def _jsconsole_enable(self):
# TODO: add public interface or make console available by default
if self._js_console is not None:
return
self._js_console = _JavascriptConsole()
frame = self.web_page.mainFrame()
frame.addToJavaScriptWindowObject('console', self._js_console)
def _jsconsole_messages(self):
# TODO: add public interface or make console available by default
if self._js_console is None:
return []
return self._js_console.messages[:]
def html(self):
""" Return HTML of the current main frame """
self.logger.log("getting HTML", min_level=2)
frame = self.web_page.mainFrame()
result = bytes(frame.toHtml().toUtf8())
self.store_har_timing("_onHtmlRendered")
return result
def png(self, width=None, height=None, b64=False):
""" Return screenshot in PNG format """
self.logger.log("getting PNG", min_level=2)
image = QImage(self.web_page.viewportSize(), QImage.Format_ARGB32)
painter = QPainter(image)
self.web_page.mainFrame().render(painter)
painter.end()
self.store_har_timing("_onScreenshotPrepared")
if width:
image = image.scaledToWidth(width, Qt.SmoothTransformation)
if height:
image = image.copy(0, 0, width, height)
b = QBuffer()
image.save(b, "png")
result = bytes(b.data())
if b64:
result = base64.b64encode(result)
self.store_har_timing("_onPngRendered")
return result
def iframes_info(self, children=True, html=True):
""" Return information about all iframes """
self.logger.log("getting iframes", min_level=3)
frame = self.web_page.mainFrame()
result = self._frame_to_dict(frame, children, html)
self.store_har_timing("_onIframesRendered")
return result
def har(self):
""" Return HAR information """
self.logger.log("getting HAR", min_level=3)
return self.web_page.har_log.todict()
def history(self):
""" Return history of 'main' HTTP requests """
self.logger.log("getting history", min_level=3)
return copy.deepcopy(self._history)
def last_http_status(self):
"""
Return HTTP status code of the currently loaded webpage
or None if it is not available.
"""
if not self._history:
return
try:
return self._history[-1]["response"]["status"]
except KeyError:
return
def _frame_to_dict(self, frame, children=True, html=True):
g = frame.geometry()
res = {
"url": unicode(frame.url().toString()),
"requestedUrl": unicode(frame.requestedUrl().toString()),
"geometry": (g.x(), g.y(), g.width(), g.height()),
"title": unicode(frame.title())
}
if html:
res["html"] = unicode(frame.toHtml())
if children:
res["childFrames"] = [
self._frame_to_dict(f, True, html)
for f in frame.childFrames()
]
res["frameName"] = unicode(frame.frameName())
return res
class WebpageRender(object):
def __init__(self,
network_manager,
splash_proxy_factory,
splash_request,
verbose=False):
self.network_manager = network_manager
self.web_view = QWebView()
self.web_page = SplashQWebPage()
self.web_page.setNetworkAccessManager(self.network_manager)
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
settings = self.web_page.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls,
True)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical,
Qt.ScrollBarAlwaysOff)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal,
Qt.ScrollBarAlwaysOff)
self.splash_request = splash_request
self.web_page.splash_request = splash_request
self.web_page.splash_proxy_factory = splash_proxy_factory
self.verbose = verbose
self.deferred = defer.Deferred()
# ======= General request/response handling:
def doRequest(self,
url,
baseurl=None,
wait_time=None,
viewport=None,
js_source=None,
js_profile=None,
console=False):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
request = QNetworkRequest()
request.setUrl(QUrl(url.decode('utf8')))
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if getattr(self.splash_request, 'pass_headers', False):
headers = self.splash_request.getAllHeaders()
for name, value in headers.items():
request.setRawHeader(name, value)
if name.lower() == 'user-agent':
self.web_page.custom_user_agent = value
if baseurl:
self._baseUrl = QUrl(baseurl.decode('utf8'))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == 'POST':
self.web_page.mainFrame().load(
request, QNetworkAccessManager.PostOperation,
self.splash_request.content.getvalue())
else:
self.web_page.mainFrame().load(request)
def close(self):
self.web_view.stop()
self.web_view.close()
self.web_page.deleteLater()
self.web_view.deleteLater()
def _requestFinished(self):
self.log("_requestFinished %s" % id(self.splash_request))
self.web_page.loadFinished.connect(self._loadFinished)
mimeType = self._reply.header(
QNetworkRequest.ContentTypeHeader).toString()
data = self._reply.readAll()
self.web_page.mainFrame().setContent(data, mimeType, self._baseUrl)
if self._reply.error():
log.msg("Error loading %s: %s" %
(self.url, self._reply.errorString()),
system='render')
self._reply.close()
self._reply.deleteLater()
def _loadFinished(self, ok):
self.log("_loadFinished %s" % id(self.splash_request))
if self.deferred.called:
# sometimes this callback is called multiple times
self.log("_loadFinished called multiple times")
return
page_ok = ok and self.web_page.errorInfo is None
maybe_redirect = not ok and self.web_page.errorInfo is None
error_loading = ok and self.web_page.errorInfo is not None
if maybe_redirect:
self.log("Redirect or other non-fatal error detected %s" %
id(self.splash_request))
# XXX: It assumes loadFinished will be called again because
# redirect happens. If redirect is detected improperly,
# loadFinished won't be called again, and Splash will return
# the result only after a timeout.
#
# FIXME: This can happen if server returned incorrect
# Content-Type header; there is no an additional loadFinished
# signal in this case.
return
if page_ok:
time_ms = int(self.wait_time * 1000)
QTimer.singleShot(time_ms, self._loadFinishedOK)
elif error_loading:
self.log("loadFinished %s: %s" %
(id(self.splash_request), str(
self.web_page.errorInfo))) #, min_level=1)
# XXX: maybe return a meaningful error page instead of generic
# error message?
self.deferred.errback(RenderError())
else:
self.log("loadFinished %s: unknown error" %
id(self.splash_request)) #, min_level=1)
self.deferred.errback(RenderError())
def _loadFinishedOK(self):
self.log("_loadFinishedOK %s" % id(self.splash_request))
try:
self._prerender()
self.deferred.callback(self._render())
except:
self.deferred.errback()
# ======= Rendering methods that subclasses can use:
def _getHtml(self):
frame = self.web_page.mainFrame()
return bytes(frame.toHtml().toUtf8())
def _getPng(self, width=None, height=None):
image = QImage(self.web_page.viewportSize(), QImage.Format_ARGB32)
painter = QPainter(image)
self.web_page.mainFrame().render(painter)
painter.end()
if width:
image = image.scaledToWidth(width, Qt.SmoothTransformation)
if height:
image = image.copy(0, 0, width, height)
b = QBuffer()
image.save(b, "png")
return bytes(b.data())
def _getIframes(self, children=True, html=True):
frame = self.web_page.mainFrame()
return self._frameToDict(frame, children, html)
def _render(self):
raise NotImplementedError()
# ======= Other helper methods:
def _setViewportSize(self, viewport):
w, h = map(int, viewport.split('x'))
size = QSize(w, h)
self.web_page.setViewportSize(size)
def _setFullViewport(self):
size = self.web_page.mainFrame().contentsSize()
if size.isEmpty():
self.log("contentsSize method doesn't work %s" %
id(self.splash_request))
self._setViewportSize(defaults.VIEWPORT_FALLBACK)
else:
self.web_page.setViewportSize(size)
def _loadJsLibs(self, frame, js_profile):
if js_profile:
for jsfile in os.listdir(js_profile):
if jsfile.endswith('.js'):
with open(os.path.join(js_profile, jsfile)) as f:
frame.evaluateJavaScript(f.read().decode('utf-8'))
def _runJS(self, js_source, js_profile):
js_output = None
js_console_output = None
if js_source:
frame = self.web_page.mainFrame()
if self.console:
js_console = JavascriptConsole()
frame.addToJavaScriptWindowObject('console', js_console)
if js_profile:
self._loadJsLibs(frame, js_profile)
ret = frame.evaluateJavaScript(js_source)
js_output = bytes(ret.toString().toUtf8())
if self.console:
js_console_output = [
bytes(s.toUtf8()) for s in js_console.messages
]
return js_output, js_console_output
def _frameToDict(self, frame, children=True, html=True):
g = frame.geometry()
res = {
"url": unicode(frame.url().toString()),
"requestedUrl": unicode(frame.requestedUrl().toString()),
"geometry": (g.x(), g.y(), g.width(), g.height()),
"title": unicode(frame.title())
}
if html:
res["html"] = unicode(frame.toHtml())
if children:
res["childFrames"] = [
self._frameToDict(f, True, html) for f in frame.childFrames()
]
res["frameName"] = unicode(frame.frameName())
return res
def _prerender(self):
if self.viewport == 'full':
self._setFullViewport()
self.js_output, self.js_console_output = self._runJS(
self.js_source, self.js_profile)
def log(self, text):
if self.verbose:
log.msg(text, system='render')
class WebpageRender(object):
"""
WebpageRender object renders a webpage: it downloads the page using
network_manager and renders it using QWebView according to options
passed to :meth:`WebpageRender.doRequest`.
This class is not used directly; its subclasses are used.
Subclasses choose how to return the result (as html, json, png).
"""
def __init__(self, network_manager, splash_proxy_factory, splash_request, verbosity):
self.network_manager = network_manager
self.web_view = QWebView()
self.web_page = SplashQWebPage(verbosity)
self.web_page.setNetworkAccessManager(self.network_manager)
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
settings = self.web_page.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls, True)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.splash_request = splash_request
self.web_page.splash_request = splash_request
self.web_page.splash_proxy_factory = splash_proxy_factory
self.verbosity = verbosity
self.deferred = defer.Deferred()
self._finished_timer = None
self._closing = False
# ======= General request/response handling:
def start(self, url, baseurl=None, wait=None, viewport=None,
js_source=None, js_profile=None, images=None, console=False,
headers=None, http_method='GET', body=None):
self.web_page.har_log.store_timing("_onStarted")
self.url = url
self.history = []
self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, images)
self.wait_time = defaults.WAIT_TIME if wait is None else wait
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
# setup logging
if self.verbosity >= 4:
self.web_page.loadStarted.connect(self._loadStarted)
self.web_page.mainFrame().loadFinished.connect(self._frameLoadFinished)
self.web_page.mainFrame().loadStarted.connect(self._frameLoadStarted)
self.web_page.mainFrame().contentsSizeChanged.connect(self._contentsSizeChanged)
if self.verbosity >= 3:
self.web_page.mainFrame().javaScriptWindowObjectCleared.connect(self._javaScriptWindowObjectCleared)
self.web_page.mainFrame().initialLayoutCompleted.connect(self._initialLayoutCompleted)
self.web_page.mainFrame().urlChanged.connect(self._urlChanged)
# do the request
request = QNetworkRequest()
request.setUrl(QUrl(url.decode('utf8')))
self._setHeaders(request, headers)
if getattr(self.splash_request, 'inspect_me', False):
# Set http method and request body from the request
http_method = self.splash_request.method
body = self.splash_request.content.getvalue()
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet,
# but in other cases it is better to set it earlier.
self._setViewportSize(self.viewport)
if baseurl:
# If baseurl is used, we download the page manually,
# then set its contents to the QWebPage and let it
# download related resources and render the result.
if http_method != 'GET':
raise NotImplementedError()
self._baseUrl = QUrl(baseurl.decode('utf8'))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
meth = OPERATION_QT_CONSTANTS[http_method]
if body is None: # PyQT doesn't support body=None
self.web_page.mainFrame().load(request, meth)
else:
self.web_page.mainFrame().load(request, meth, body)
def render(self):
"""
This method is called to get the result after the requested page is
downloaded and rendered. Subcalles should implement it to customize
which data to return.
"""
raise NotImplementedError()
def close(self):
"""
This method is called by a Pool after the rendering is done and
the WebpageRender object is no longer needed.
"""
self._closing = True
self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)
self.web_view.stop()
self.web_view.close()
self.web_page.deleteLater()
self.web_view.deleteLater()
def _setHeaders(self, request, headers):
""" Set HTTP headers for the ``request``. """
if isinstance(headers, dict):
headers = headers.items()
for name, value in headers or []:
request.setRawHeader(name, value)
if name.lower() == 'user-agent':
self.web_page.custom_user_agent = value
def _requestFinished(self):
"""
This method is called when ``baseurl`` is used and a
reply for the first request is received.
"""
self.log("_requestFinished %s" % id(self.splash_request))
self.web_page.loadFinished.connect(self._loadFinished)
mimeType = self._reply.header(QNetworkRequest.ContentTypeHeader).toString()
data = self._reply.readAll()
self.web_page.mainFrame().setContent(data, mimeType, self._baseUrl)
if self._reply.error():
self.log("Error loading %s: %s" % (self.url, self._reply.errorString()), min_level=1)
self._reply.close()
self._reply.deleteLater()
def _loadFinished(self, ok):
"""
This method is called when a QWebPage finished loading its contents.
"""
if self._closing:
self.log("loadFinished is ignored because WebpageRender is closing", min_level=3)
return
if self.deferred.called:
# sometimes this callback is called multiple times
self.log("loadFinished called multiple times", min_level=1)
return
page_ok = ok and self.web_page.errorInfo is None
maybe_redirect = not ok and self.web_page.errorInfo is None
error_loading = ok and self.web_page.errorInfo is not None
if maybe_redirect:
self.log("Redirect or other non-fatal error detected %s" % id(self.splash_request))
# XXX: It assumes loadFinished will be called again because
# redirect happens. If redirect is detected improperly,
# loadFinished won't be called again, and Splash will return
# the result only after a timeout.
#
# FIXME: This can happen if server returned incorrect
# Content-Type header; there is no an additional loadFinished
# signal in this case.
return
if page_ok: # or maybe_redirect:
if self.wait_time == 0:
self.log("loadFinished %s; not waiting" % (id(self.splash_request)))
self._loadFinishedOK()
else:
time_ms = int(self.wait_time * 1000)
self.log("loadFinished %s; waiting %sms" % (id(self.splash_request), time_ms))
if self._finished_timer is not None:
raise Exception("timer is not None!")
self._finished_timer = QTimer()
self._finished_timer.setSingleShot(True)
self._finished_timer.timeout.connect(self._loadFinishedOK)
self._finished_timer.start(time_ms)
elif error_loading:
self.log("loadFinished %s: %s" % (id(self.splash_request), str(self.web_page.errorInfo)), min_level=1)
# XXX: maybe return a meaningful error page instead of generic
# error message?
self.deferred.errback(RenderError())
else:
self.log("loadFinished %s: unknown error" % id(self.splash_request), min_level=1)
self.deferred.errback(RenderError())
def _loadFinishedOK(self):
self._finished_timer = None
self.log("_loadFinishedOK %s" % id(self.splash_request))
if self._closing:
self.log("loadFinishedOK is ignored because WebpageRender is closing", min_level=3)
return
self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)
self.web_view.stop()
self.web_page.har_log.store_timing("_onPrepareStart")
try:
self._prepareRender()
self.deferred.callback(self.render())
except:
self.deferred.errback()
def _frameLoadFinished(self, ok):
self.log("mainFrame().LoadFinished %s %s" % (id(self.splash_request), ok), min_level=4)
def _loadStarted(self):
self.log("loadStarted %s" % id(self.splash_request), min_level=4)
def _urlChanged(self, url):
cause_ev = self.web_page.har_log._prev_entry(unicode(url.toString()), -1)
if cause_ev:
self.history.append(without_private(cause_ev.data))
msg = "mainFrame().urlChanged %s: %s" % (id(self.splash_request), qurl2ascii(url))
self.log(msg, min_level=3)
if self._finished_timer is not None:
self.log("Cancelling wait timer %s" % (id(self.splash_request)))
self._finished_timer.stop()
self._finished_timer = None
def _frameLoadStarted(self):
self.log("mainFrame().loadStarted %s" % id(self.splash_request), min_level=4)
def _initialLayoutCompleted(self):
self.log("mainFrame().initialLayoutCompleted %s" % id(self.splash_request), min_level=3)
def _javaScriptWindowObjectCleared(self):
self.log("mainFrame().javaScriptWindowObjectCleared %s" % id(self.splash_request), min_level=3)
def _contentsSizeChanged(self):
self.log("mainFrame().contentsSizeChanged %s" % id(self.splash_request), min_level=4)
def _repaintRequested(self):
self.log("mainFrame().repaintRequested %s" % id(self.splash_request), min_level=4)
# ======= Rendering methods that subclasses can use:
def _getHtml(self):
self.log("getting HTML %s" % id(self.splash_request))
frame = self.web_page.mainFrame()
result = bytes(frame.toHtml().toUtf8())
self.web_page.har_log.store_timing("_onHtmlRendered")
return result
def _getPng(self, width=None, height=None, b64=False):
self.log("getting PNG %s" % id(self.splash_request))
image = QImage(self.web_page.viewportSize(), QImage.Format_ARGB32)
painter = QPainter(image)
self.web_page.mainFrame().render(painter)
painter.end()
self.web_page.har_log.store_timing("_onScreenshotPrepared")
if width:
image = image.scaledToWidth(width, Qt.SmoothTransformation)
if height:
image = image.copy(0, 0, width, height)
b = QBuffer()
image.save(b, "png")
result = bytes(b.data())
if b64:
result = base64.b64encode(result)
self.web_page.har_log.store_timing("_onPngRendered")
return result
def _getIframes(self, children=True, html=True):
self.log("getting iframes %s" % id(self.splash_request), min_level=3)
frame = self.web_page.mainFrame()
result = self._frameToDict(frame, children, html)
self.web_page.har_log.store_timing("_onIframesRendered")
return result
def _getHistory(self):
self.log("getting history %s" % id(self.splash_request), min_level=3)
hist = copy.deepcopy(self.history)
for entry in hist:
if entry is not None:
del entry['request']['queryString']
return hist
def _getHAR(self):
self.log("getting HAR %s" % id(self.splash_request), min_level=3)
return self.web_page.har_log.todict()
# ======= Other helper methods:
def _setViewportSize(self, size):
if not isinstance(size, QSize):
w, h = map(int, size.split('x'))
size = QSize(w, h)
self.web_page.setViewportSize(size)
w, h = int(size.width()), int(size.height())
self.log("viewport size for %s is set to %sx%s" % (id(self.splash_request), w, h))
def _setFullViewport(self):
size = self.web_page.mainFrame().contentsSize()
if size.isEmpty():
self.log("contentsSize method doesn't work %s" % id(self.splash_request), min_level=1)
self._setViewportSize(defaults.VIEWPORT_FALLBACK)
else:
self._setViewportSize(size)
self.web_page.har_log.store_timing("_onFullViewportSet")
def _loadJsLibs(self, frame, js_profile):
if js_profile:
for jsfile in os.listdir(js_profile):
if jsfile.endswith('.js'):
with open(os.path.join(js_profile, jsfile)) as f:
frame.evaluateJavaScript(f.read().decode('utf-8'))
def _runJS(self, js_source, js_profile):
js_output = None
js_console_output = None
if js_source:
frame = self.web_page.mainFrame()
if self.console:
js_console = JavascriptConsole()
frame.addToJavaScriptWindowObject('console', js_console)
if js_profile:
self._loadJsLibs(frame, js_profile)
ret = frame.evaluateJavaScript(js_source)
js_output = bytes(ret.toString().toUtf8())
if self.console:
js_console_output = [bytes(s.toUtf8()) for s in js_console.messages]
self.web_page.har_log.store_timing('_onCustomJsExecuted')
return js_output, js_console_output
def _frameToDict(self, frame, children=True, html=True):
g = frame.geometry()
res = {
"url": unicode(frame.url().toString()),
"requestedUrl": unicode(frame.requestedUrl().toString()),
"geometry": (g.x(), g.y(), g.width(), g.height()),
"title": unicode(frame.title())
}
if html:
res["html"] = unicode(frame.toHtml())
if children:
res["childFrames"] = [self._frameToDict(f, True, html) for f in frame.childFrames()]
res["frameName"] = unicode(frame.frameName())
return res
def _prepareRender(self):
if self.viewport == 'full':
self._setFullViewport()
self.js_output, self.js_console_output = self._runJS(self.js_source, self.js_profile)
def log(self, text, min_level=2):
if self.verbosity >= min_level:
if isinstance(text, unicode):
text = text.encode('unicode-escape').decode('ascii')
log.msg(text, system='render')
class SchedaEdificio(QMainWindow, MappingOne2One, Ui_SchedaEdificio):
def __init__(self, parent=None, schedaID=None):
QMainWindow.__init__(self, parent)
MappingOne2One.__init__(self, "SCHEDA_EDIFICIO")
self.setAttribute(Qt.WA_DeleteOnClose)
self.setupUi(self)
self.sectionsStacked.setCurrentIndex(0)
# carica i widget multivalore con i valori delle relative tabelle
tablesDict = {
self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ.ZZ_PROPRIETAID:
AutomagicallyUpdater.ZZTable("ZZ_PROPRIETA")
}
self.setupTablesUpdater(tablesDict)
self.loadTables()
# mappa i widget con i campi delle tabelle
childrenList = [
self.PRINCIPALE, self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ,
self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ.ZZ_PROPRIETAID,
self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ.NUM_UNITA_IMMOBILIARI,
self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ.PARTICELLE_CATASTALI,
self.UNITA_VOLUMETRICHE, self.INTERVENTI,
self.STATO_UTILIZZO_EDIFICIOID, self.CARATTERISTICHE_STRUTTURALI,
self.CARATTERISTICHE_ARCHITETTONICHE_EDIFICIOID, self.FOTO
]
self.setupValuesUpdater(childrenList)
self.connect(self.sectionsList, SIGNAL("itemSelectionChanged()"),
self.currentSectionChanged)
self.connect(self.PRINCIPALE.printBtn, SIGNAL("clicked()"),
self.stampaScheda)
# aggiorna il titolo della scheda con l'indirizzo del primo tab indirizzi
indirizzo = self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ.LOCALIZZAZIONE_EDIFICIO_INDIRIZZO_VIA.firstTab
self.connect(indirizzo, SIGNAL("indirizzoChanged"),
self.aggiornaTitolo)
# carica i dati della scheda
self.setupLoader(schedaID)
self.aggiornaTitolo()
def currentSectionChanged(self):
if self.sectionsList.currentRow() < 0:
return
self.sectionsStacked.setCurrentIndex(self.sectionsList.currentRow())
def aggiornaTitolo(self):
indirizzo = self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ.LOCALIZZAZIONE_EDIFICIO_INDIRIZZO_VIA.firstTab
comune = indirizzo.ZZ_COMUNIISTATCOM.currentText()
provincia = indirizzo.ZZ_PROVINCEISTATPROV.currentText()
via = indirizzo.VIA.currentText()
civico = indirizzo.NUMERI_CIVICI.rowToString()
from WdgLocalizzazioneIndirizzi import WdgLocalizzazioneIndirizzi
indirizzo_non_valido = WdgLocalizzazioneIndirizzi.INDIRIZZO_NON_VALIDO
indirizzo_non_inserito = WdgLocalizzazioneIndirizzi.INDIRIZZO_NON_INSERITO
if comune != '':
via = u"%s, %s" % (
via, civico
) if via != '' and civico != '' else indirizzo_non_inserito
titolo = u"%s - %s (%s)" % (via, comune, provincia)
else:
titolo = indirizzo_non_valido
self.setWindowTitle(titolo)
def getTitoloStampa(self):
indirizzo = self.LOCALIZZAZIONE_EDIFICIOIDLOCALIZZ.LOCALIZZAZIONE_EDIFICIO_INDIRIZZO_VIA.firstTab
istatcom = self.getValue(indirizzo.ZZ_COMUNIISTATCOM)
via = indirizzo.VIA.currentText()
civico = indirizzo.NUMERI_CIVICI.rowToString()
if istatcom != None and via != '' and civico != '':
return u"%s - %s - %s" % (istatcom, via, civico)
return self._ID
def stampaScheda(self, preview=True):
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
self.PRINCIPALE.printBtn.setEnabled(False)
self._previewOnPrinting = preview
settings = QSettings()
self._printMode, _ = AutomagicallyUpdater.printBehavior()
if self._printMode in (QPrinter.PdfFormat, QPrinter.NativeFormat):
# create a webview to load the HTML
from PyQt4.QtWebKit import QWebView
self._webView = QWebView(self)
self._webView.setVisible(False)
QObject.connect(self._webView, SIGNAL("loadFinished(bool)"),
self.webViewLoadFinished)
# generate the HTML and load it into the webview
html = self.toHtml()
self._webView.setHtml(html)
else: # print to HTML files
outputDir = AutomagicallyUpdater._getLastUsedDir('pdf')
if preview:
QApplication.restoreOverrideCursor()
outputDir = QFileDialog.getExistingDirectory(
self, "Salvataggio scheda", outputDir,
QFileDialog.ShowDirsOnly)
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if outputDir == "":
return self.onPrintFinished(True)
AutomagicallyUpdater._setLastUsedDir('pdf', outputDir)
# create the dir that will contain the html file within the output dir
outputDir = QDir(outputDir)
outputPath = outputDir.filePath(self._ID)
if not outputDir.exists(self._ID) and not outputDir.mkdir(
self._ID):
QMessageBox.warning(
self, "RT Omero",
u"Impossibile creare il percorso '%s'" % outputPath)
return self.onPrintFinished(False)
# generate the HTML using outputPath as directory for the images/resources
html = self.toHtml(outputPath)
htmlFile = QDir(outputPath).filePath("index.html")
with open(htmlFile.toUtf8(), 'w') as fout:
fout.write(html)
self.onPrintFinished(True)
#if preview:
# QDesktopServices.openUrl( QUrl.fromLocalFile( htmlFile ) )
def webViewLoadFinished(self, ok):
if not ok:
return self.onPrintFinished(False)
# get the instance of the printer
from ManagerWindow import ManagerWindow
printer = ManagerWindow.instance.getPrinter()
printer.setDocName(self.getTitoloStampa())
if self._printMode == QPrinter.PdfFormat:
# set the output format and filename
lastDir = AutomagicallyUpdater._getLastUsedDir('pdf')
outFn = QDir(lastDir).filePath(u"%s.pdf" % self.getTitoloStampa())
printer.setOutputFileName(outFn)
if self._previewOnPrinting:
printDlg = QPrintPreviewDialog(printer, self)
QObject.connect(printDlg, SIGNAL("paintRequested(QPrinter *)"),
self._webView.print_)
QApplication.restoreOverrideCursor()
ret = printDlg.exec_()
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if ret:
if self._printMode == QPrinter.PdfFormat:
AutomagicallyUpdater._setLastUsedDir(
'pdf', printer.outputFileName())
printDlg.deleteLater()
del printDlg
elif self._printMode == QPrinter.NativeFormat:
printDlg = QPrintDialog(printer, self)
QApplication.restoreOverrideCursor()
ret = printDlg.exec_()
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if ret:
self._webView.print_(printer)
else: # print to PDF without asking to the user
self._webView.print_(printer)
return self.onPrintFinished(ok)
def onPrintFinished(self, ok):
# remove temporary files, i.e. images/resources used in the HTML
from Utils import TemporaryFile
TemporaryFile.delAllFiles(TemporaryFile.KEY_SCHEDAEDIFICIO2HTML)
if self._printMode in (QPrinter.PdfFormat, QPrinter.NativeFormat):
self._webView.deleteLater()
del self._webView
self.PRINCIPALE.printBtn.setEnabled(True)
QApplication.restoreOverrideCursor()
self.emit(SIGNAL("printFinished"), ok, self._ID)
return ok
def creaStralcioCartografico(self,
size,
scale,
ext="png",
factor=1,
outpath=None):
from ManagerWindow import ManagerWindow
def renderScaleLabel(painter, scale, factor=1):
text = "1:%s" % (scale * factor)
fontSize = 10 * factor
font = QFont("helvetica", fontSize)
painter.setFont(font)
fontMetrics = QFontMetrics(font)
margin = 20 * factor
bufferSize = 1
backColor = Qt.white
foreColor = Qt.black
# first the buffer
painter.setPen(backColor)
fontWidth = fontMetrics.width(text)
fontHeight = fontMetrics.height()
for i in range(-bufferSize, bufferSize + 1):
for j in range(-bufferSize, bufferSize + 1):
painter.drawText(i + margin, j + margin, text)
# then the text itself
painter.setPen(foreColor)
painter.drawText(margin, margin, text)
def createStralcioUsingCanvas(filename,
size,
scale,
ext="png",
factor=1):
# get the reference to the main canvas and its renderer
mainCanvas = ManagerWindow.instance.iface.mapCanvas()
mainRenderer = mainCanvas.mapRenderer()
# create a new map canvas and setup it
canvas = qgis.gui.QgsMapCanvas(
ManagerWindow.instance.iface.mainWindow())
canvas.setCanvasColor(Qt.white)
canvas.setFixedSize(size.width(), size.height())
canvas.setRenderFlag(False)
settings = QSettings()
canvas.enableAntiAliasing(
settings.value("/qgis/enable_anti_aliasing", False).toBool())
canvas.useImageToRender(
settings.value("/qgis/use_qimage_to_render", False).toBool())
canvas.mapRenderer().setDestinationSrs(
mainRenderer.destinationSrs())
canvas.mapRenderer().setMapUnits(mainRenderer.mapUnits())
try:
canvasLayers = []
# add WMS layers
for order, rlid in sorted(ManagerWindow.RLID_WMS.iteritems()):
layer = QgsMapLayerRegistry.instance().mapLayer(rlid)
if layer != None and ManagerWindow.instance.iface.legendInterface(
).isLayerVisible(layer):
cl = qgis.gui.QgsMapCanvasLayer(layer)
canvasLayers.insert(0, cl)
# add other layers
layers = [
ManagerWindow.VLID_GEOM_ORIG,
ManagerWindow.VLID_GEOM_MODIF, ManagerWindow.VLID_FOTO
]
for vlid in layers:
layer = QgsMapLayerRegistry.instance().mapLayer(vlid)
if layer != None:
layer.removeSelection()
cl = qgis.gui.QgsMapCanvasLayer(layer)
canvasLayers.insert(0, cl)
canvas.setLayerSet(canvasLayers)
# XXX: why? it's needed to update the extent of the other canvas
canvas.show()
mainCanvas.setRenderFlag(True)
# select the geometries
layerModif = QgsMapLayerRegistry.instance().mapLayer(
ManagerWindow.VLID_GEOM_MODIF)
if layerModif != None:
query = AutomagicallyUpdater.Query(
"SELECT gmod.ROWID FROM GEOMETRIE_RILEVATE_NUOVE_O_MODIFICATE AS gmod JOIN SCHEDA_UNITA_VOLUMETRICA AS suv ON gmod.ID_UV_NEW = suv.GEOMETRIE_RILEVATE_NUOVE_O_MODIFICATEID_UV_NEW WHERE SCHEDA_EDIFICIOID = ?",
[self._ID]).getQuery()
if query.exec_():
selIds = []
while query.next():
selIds.append(query.value(0).toInt()[0])
layerModif.setSelectedFeatures(selIds)
canvas.zoomToSelected(layerModif)
# zoom to scale
canvas.zoomScale(scale)
# save the map to a file
eventLoopHandler = [QEventLoop(self)]
def savePainter(p, scale, evlHandler):
renderScaleLabel(p, scale)
evlHandler[0].quit()
evlHandler[0].deleteLater()
del evlHandler[0]
onRenderFunc = lambda x: savePainter(x, scale, eventLoopHandler
)
QObject.connect(canvas, SIGNAL("renderComplete(QPainter *)"),
onRenderFunc)
canvas.setRenderFlag(True)
if len(eventLoopHandler) > 0:
eventLoopHandler[0].exec_(
QEventLoop.ExcludeUserInputEvents)
QObject.disconnect(canvas,
SIGNAL("renderComplete(QPainter *)"),
onRenderFunc)
canvas.saveAsImage(filename, None, ext.upper())
extent = canvas.extent()
# remove the wordfile create by canvas.saveAsImage()
wordfile = QFile(filename[:-4] + filename[-4:].upper() + "w")
if wordfile.exists():
wordfile.remove()
finally:
canvas.hide()
canvas.deleteLater()
del canvas
return filename, extent
def createStralcioUsingRenderer(filename,
size,
scale,
ext="png",
factor=1):
# create the output image and pre-fill it
image = QPixmap(size)
image.fill(QColor(255, 255, 255, 255))
# get the reference to the main canvas and its renderer
mainCanvas = ManagerWindow.instance.iface.mapCanvas()
mainRenderer = mainCanvas.mapRenderer()
# create a new renderer and setup it
mapRenderer = qgis.core.QgsMapRenderer()
mapRenderer.setOutputSize(size, image.logicalDpiX())
mapRenderer.setDestinationSrs(mainRenderer.destinationSrs())
mapRenderer.setMapUnits(mainRenderer.mapUnits())
mapRenderer.setProjectionsEnabled(
mainRenderer.hasCrsTransformEnabled())
# add layers to renderer layer set
layerIds = []
# add WMS layers
for order, rlid in sorted(ManagerWindow.RLID_WMS.iteritems()):
layer = QgsMapLayerRegistry.instance().mapLayer(rlid)
if layer != None and ManagerWindow.instance.iface.legendInterface(
).isLayerVisible(layer):
lid = layer.id() if hasattr(
layer,
'id') else layer.getLayerID() # old API compatibility
layerIds.insert(0, lid)
# add other layers
layers = [
ManagerWindow.VLID_GEOM_ORIG, ManagerWindow.VLID_GEOM_MODIF,
ManagerWindow.VLID_FOTO
]
for vlid in layers:
layer = QgsMapLayerRegistry.instance().mapLayer(vlid)
if layer != None:
layer.removeSelection()
lid = layer.id() if hasattr(
layer,
'id') else layer.getLayerID() # old API compatibility
layerIds.insert(0, lid)
mapRenderer.setLayerSet(layerIds)
# select the geometries
layerModif = QgsMapLayerRegistry.instance().mapLayer(
ManagerWindow.VLID_GEOM_MODIF)
if layerModif != None:
query = AutomagicallyUpdater.Query(
"SELECT gmod.ROWID FROM GEOMETRIE_RILEVATE_NUOVE_O_MODIFICATE AS gmod JOIN SCHEDA_UNITA_VOLUMETRICA AS suv ON gmod.ID_UV_NEW = suv.GEOMETRIE_RILEVATE_NUOVE_O_MODIFICATEID_UV_NEW WHERE SCHEDA_EDIFICIOID = ?",
[self._ID]).getQuery()
if query.exec_():
selIds = []
while query.next():
selIds.append(query.value(0).toInt()[0])
layerModif.setSelectedFeatures(selIds)
mapRenderer.setExtent(layerModif.boundingBoxOfSelected())
# zoom at the scale
extent = mapRenderer.extent()
extent.scale(float(scale) / mapRenderer.scale())
mapRenderer.setExtent(extent)
# render now!
painter = QPainter()
painter.begin(image)
#settings = QSettings()
#antiAliasingEnabled = settings.value( "/qgis/enable_anti_aliasing", False ).toBool()
#painter.setRenderHints( QPainter.RenderHints() | (QPainter.Antialiasing if antiAliasingEnabled else 0) )
mapRenderer.render(painter)
renderScaleLabel(painter, scale)
painter.end()
del painter
mapRenderer.deleteLater()
del mapRenderer
# save the image to a file
image.save(filename, ext.upper())
del image
return filename, extent
if outpath is None:
# get a new temp file
from Utils import TemporaryFile
tmp = TemporaryFile.getNewFile(
TemporaryFile.KEY_SCHEDAEDIFICIO2HTML, ext)
if not tmp.open():
TemporaryFile.delFile(TemporaryFile.KEY_SCHEDAEDIFICIO2HTML,
ext)
return QString(), None
filename = unicode(tmp.fileName())
tmp.close()
else:
filename = u"%s/stralcio.%s" % (outpath, ext)
# override the canvas render flag
mainCanvas = ManagerWindow.instance.iface.mapCanvas()
prevRenderFlag = mainCanvas.renderFlag()
mainCanvas.setRenderFlag(False)
# override the selection color
prevColor = QgsRenderer.selectionColor()
newColor = QColor(Qt.yellow)
newColor.setAlpha(127)
QgsRenderer.setSelectionColor(newColor)
# set layers visible
legend = ManagerWindow.instance.iface.legendInterface()
layerOrig = QgsMapLayerRegistry.instance().mapLayer(
ManagerWindow.VLID_GEOM_ORIG)
if layerOrig != None:
prevOrigState = ManagerWindow.instance.iface.legendInterface(
).isLayerVisible(layerOrig)
ManagerWindow.instance.iface.legendInterface().setLayerVisible(
layerOrig, True)
layerModif = QgsMapLayerRegistry.instance().mapLayer(
ManagerWindow.VLID_GEOM_MODIF)
if layerModif != None:
prevModifState = ManagerWindow.instance.iface.legendInterface(
).isLayerVisible(layerModif)
prevSelection = layerModif.selectedFeaturesIds()
ManagerWindow.instance.iface.legendInterface().setLayerVisible(
layerModif, True)
try:
if True:
# XXX: why? the output image seems to be generated at a wrong scale using a new renderer
filename, extent = createStralcioUsingCanvas(
filename, size, scale, ext, factor)
else:
filename, extent = createStralcioUsingRenderer(
filename, size, scale, ext, factor)
finally:
# restore the original layers' state and selection
if layerOrig != None:
legend.setLayerVisible(layerOrig, prevOrigState)
if layerModif != None:
legend.setLayerVisible(layerModif, prevModifState)
layerModif.setSelectedFeatures(prevSelection)
# restore the canvas original state and selection color
QgsRenderer.setSelectionColor(prevColor)
mainCanvas.setRenderFlag(prevRenderFlag)
return filename, extent
def setMinimized(self, minimize=True):
if minimize:
self.setWindowState(self.windowState() | Qt.WindowMinimized)
else:
self.setWindowState(self.windowState() & ~Qt.WindowMinimized)
self.raise_()
def closeEvent(self, event):
# rimuovi i file temporanei collegati alla scheda
from Utils import TemporaryFile
TemporaryFile.delAllFiles(TemporaryFile.KEY_SCHEDAEDIFICIO)
try:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
ConnectionManager.startTransaction()
# effettua il salvataggio della scheda
self.save()
except ConnectionManager.AbortedException, e:
QMessageBox.critical(self, "Errore", e.toString())
return
finally:
class WebpageRender(object):
def __init__(self, network_manager):
self.network_manager = network_manager
self.web_view = QWebView()
self.web_page = SplashQWebPage()
self.web_page.setNetworkAccessManager(self.network_manager)
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
settings = self.web_view.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
def doRequest(self, url, baseurl=None, wait_time=None):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.deferred = defer.Deferred()
request = QNetworkRequest()
request.setUrl(QUrl(url))
if baseurl:
self._baseUrl = QUrl(baseurl)
self.network_manager.finished.connect(self._requestFinished)
self.network_manager.get(request)
else:
self.web_page.loadFinished.connect(self._loadFinished)
self.web_page.mainFrame().load(request)
def close(self):
self.web_view.stop()
self.web_view.close()
self.web_page.deleteLater()
self.web_view.deleteLater()
self.network_manager.deleteLater()
def _requestFinished(self, reply):
self.web_page.networkAccessManager().finished.disconnect(self._requestFinished)
self.web_view.loadFinished.connect(self._loadFinished)
mimeType = reply.header(QNetworkRequest.ContentTypeHeader).toString()
self.web_view.page().mainFrame().setContent(reply.readAll(), mimeType, self._baseUrl)
def _loadFinished(self, ok):
if self.deferred.called:
return
if ok:
time_ms = int(self.wait_time * 1000)
QTimer.singleShot(time_ms, self._loadFinishedOK)
else:
self.deferred.errback(RenderError())
def _loadFinishedOK(self):
if self.deferred.called:
return
try:
self.deferred.callback(self._render())
except:
self.deferred.errback()
def _getHtml(self):
frame = self.web_view.page().mainFrame()
return bytes(frame.toHtml().toUtf8())
def _getPng(self, width=None, height=None, viewport=None):
self._setViewportSize(viewport)
image = QImage(self.web_page.viewportSize(), QImage.Format_ARGB32)
painter = QPainter(image)
self.web_page.mainFrame().render(painter)
painter.end()
if width:
image = image.scaledToWidth(width, Qt.SmoothTransformation)
if height:
image = image.copy(0, 0, width, height)
b = QBuffer()
image.save(b, "png")
return bytes(b.data())
def _getIframes(self, children=True, html=True):
frame = self.web_view.page().mainFrame()
return self._frameToDict(frame, children, html)
def _setViewportSize(self, viewport=None):
viewport = defaults.VIEWPORT if viewport is None else viewport
if viewport == 'full':
size = self.web_page.mainFrame().contentsSize()
if size.isEmpty(): # sometimes contentsSize doesn't work
w, h = map(int, defaults.VIEWPORT_FALLBACK.split('x'))
size = QSize(w, h)
else:
w, h = map(int, viewport.split('x'))
size = QSize(w, h)
self.web_page.setViewportSize(size)
def _frameToDict(self, frame, children=True, html=True):
g = frame.geometry()
res = {
"url": unicode(frame.url().toString()),
"requestedUrl": unicode(frame.requestedUrl().toString()),
"geometry": (g.x(), g.y(), g.width(), g.height()),
"title": unicode(frame.title())
}
if html:
res["html"] = unicode(frame.toHtml())
if children:
res["childFrames"] = [self._frameToDict(f, True, html) for f in frame.childFrames()]
res["frameName"] = unicode(frame.frameName())
return res
def _render(self):
raise NotImplementedError()
class BrowserTab(object):
"""
An object for controlling a single browser tab (QWebView).
It is created by splash.pool.Pool. Pool attaches to tab's deferred
and waits until either a callback or an errback is called, then destroys
a BrowserTab.
"""
def __init__(self, network_manager, splash_proxy_factory, verbosity,
render_options):
""" Create a new browser tab. """
self.deferred = defer.Deferred()
self.network_manager = network_manager
self.verbosity = verbosity
self._uid = render_options.get_uid()
self._closing = False
self._default_headers = None
self._active_timers = set()
self._timers_to_cancel_on_redirect = weakref.WeakKeyDictionary(
) # timer: callback
self._timers_to_cancel_on_error = weakref.WeakKeyDictionary(
) # timer: callback
self._js_console = None
self._history = []
self._init_webpage(verbosity, network_manager, splash_proxy_factory,
render_options)
self._setup_logging(verbosity)
def _init_webpage(self, verbosity, network_manager, splash_proxy_factory,
render_options):
""" Create and initialize QWebPage and QWebView """
self.web_page = SplashQWebPage(verbosity)
self.web_page.setNetworkAccessManager(network_manager)
self.web_page.splash_proxy_factory = splash_proxy_factory
self.web_page.render_options = render_options
self._set_default_webpage_options(self.web_page)
self._setup_webpage_events()
self.web_view = QWebView()
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
def _set_default_webpage_options(self, web_page):
"""
Set QWebPage options.
TODO: allow to customize them.
"""
settings = web_page.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls,
True)
web_page.mainFrame().setScrollBarPolicy(Qt.Vertical,
Qt.ScrollBarAlwaysOff)
web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal,
Qt.ScrollBarAlwaysOff)
def _setup_logging(self, verbosity):
""" Setup logging of various events """
self.logger = _BrowserTabLogger(
uid=self._uid,
web_page=self.web_page,
verbosity=verbosity,
)
self.logger.enable()
def _setup_webpage_events(self):
self._load_finished = WrappedSignal(
self.web_page.mainFrame().loadFinished)
self.web_page.mainFrame().loadFinished.connect(self._on_load_finished)
self.web_page.mainFrame().urlChanged.connect(self._on_url_changed)
def return_result(self, result):
""" Return a result to the Pool. """
if self.result_already_returned():
self.logger.log("error: result is already returned", min_level=1)
self.deferred.callback(result)
# self.deferred = None
def return_error(self, error=None):
""" Return an error to the Pool. """
if self.result_already_returned():
self.logger.log("error: result is already returned", min_level=1)
self.deferred.errback(error)
# self.deferred = None
def result_already_returned(self):
""" Return True if an error or a result is already returned to Pool """
return self.deferred.called
def set_default_headers(self, headers):
""" Set default HTTP headers """
self._default_headers = headers
def set_images_enabled(self, enabled):
self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages,
enabled)
def set_viewport(self, size):
"""
Set viewport size.
If size is "full" viewport size is detected automatically.
If can also be "<width>x<height>".
"""
if size == 'full':
size = self.web_page.mainFrame().contentsSize()
if size.isEmpty():
self.logger.log("contentsSize method doesn't work %s",
min_level=1)
size = defaults.VIEWPORT_FALLBACK
if not isinstance(size, QSize):
w, h = map(int, size.split('x'))
size = QSize(w, h)
self.web_page.setViewportSize(size)
w, h = int(size.width()), int(size.height())
self.logger.log("viewport size is set to %sx%s" % (w, h), min_level=2)
return w, h
@property
def url(self):
""" Current URL """
return unicode(self.web_page.mainFrame().url().toString())
def go(self,
url,
callback,
errback,
baseurl=None,
http_method='GET',
body=None):
"""
Go to an URL. This is similar to entering an URL in
address tab and pressing Enter.
"""
self.store_har_timing("_onStarted")
if baseurl:
# If baseurl is used, we download the page manually,
# then set its contents to the QWebPage and let it
# download related resources and render the result.
if http_method != 'GET':
raise NotImplementedError()
request = self._create_request(url)
request.setOriginatingObject(self.web_page.mainFrame())
# TODO / FIXME: add support for multiple replies
# or discard/cancel previous replies
self._reply = self.network_manager.get(request)
cb = functools.partial(
self._on_baseurl_request_finished,
callback=callback,
errback=errback,
baseurl=baseurl,
url=url,
)
self._reply.finished.connect(cb)
else:
# if not self._goto_callbacks.isempty():
# self.logger.log("Only a single concurrent 'go' request is supported. "
# "Previous go requests will be cancelled.", min_level=1)
# # When a new URL is loaded to mainFrame an errback will
# # be called, so we're not cancelling this callback manually.
callback_id = self._load_finished.connect(
self._on_goto_load_finished,
callback=callback,
errback=errback,
)
self.logger.log("callback %s is connected to loadFinished" %
callback_id,
min_level=3)
self._load_url_to_mainframe(url, http_method, body)
def stop_loading(self):
"""
Stop loading of the current page and all pending page
refresh/redirect requests.
"""
self.logger.log("stop_loading", min_level=2)
self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)
self.web_view.stop()
def _close(self):
""" Destroy this tab """
self._closing = True
self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)
self.web_view.stop()
self.web_view.close()
self.web_page.deleteLater()
self.web_view.deleteLater()
self._cancel_all_timers()
@skip_if_closing
def _on_load_finished(self, ok):
if self.web_page.maybe_redirect(ok):
self.logger.log("Redirect or other non-fatal error detected",
min_level=2)
return
if self.web_page.is_ok(ok): # or maybe_redirect:
self.logger.log("loadFinished: ok", min_level=2)
else:
self._cancel_timers(self._timers_to_cancel_on_error)
if self.web_page.error_loading(ok):
self.logger.log("loadFinished: %s" %
(str(self.web_page.error_info)),
min_level=1)
else:
self.logger.log("loadFinished: unknown error", min_level=1)
def _on_baseurl_request_finished(self, callback, errback, baseurl, url):
"""
This method is called when ``baseurl`` is used and a
reply for the first request is received.
"""
self.logger.log("baseurl_request_finished", min_level=2)
callback_id = self._load_finished.connect(
self._on_goto_load_finished,
callback=callback,
errback=errback,
)
self.logger.log("callback %s is connected to loadFinished" %
callback_id,
min_level=3)
baseurl = QUrl(baseurl)
mimeType = self._reply.header(
QNetworkRequest.ContentTypeHeader).toString()
data = self._reply.readAll()
self.web_page.mainFrame().setContent(data, mimeType, baseurl)
if self._reply.error():
self.logger.log("Error loading %s: %s" %
(url, self._reply.errorString()),
min_level=1)
self._reply.close()
self._reply.deleteLater()
def _load_url_to_mainframe(self, url, http_method, body=None):
request = self._create_request(url)
meth = OPERATION_QT_CONSTANTS[http_method]
if body is None: # PyQT doesn't support body=None
self.web_page.mainFrame().load(request, meth)
else:
self.web_page.mainFrame().load(request, meth, body)
def _create_request(self, url):
request = QNetworkRequest()
request.setUrl(QUrl(url))
self._set_request_headers(request, self._default_headers)
return request
@skip_if_closing
def _on_goto_load_finished(self, ok, callback, errback, callback_id):
"""
This method is called when a QWebPage finishes loading its contents.
"""
if self.web_page.maybe_redirect(ok):
# XXX: It assumes loadFinished will be called again because
# redirect happens. If redirect is detected improperly,
# loadFinished won't be called again, and Splash will return
# the result only after a timeout.
return
self.logger.log("loadFinished: disconnecting callback %s" %
callback_id,
min_level=3)
self._load_finished.disconnect(callback_id)
if self.web_page.is_ok(ok):
callback()
elif self.web_page.error_loading(ok):
# XXX: maybe return a meaningful error page instead of generic
# error message?
errback()
# errback(RenderError())
else:
errback()
# errback(RenderError())
def _set_request_headers(self, request, headers):
""" Set HTTP headers for the request. """
if isinstance(headers, dict):
headers = headers.items()
for name, value in headers or []:
request.setRawHeader(name, value)
if name.lower() == 'user-agent':
self.web_page.custom_user_agent = value
def wait(self, time_ms, callback, onredirect=None, onerror=None):
"""
Wait for time_ms, then run callback.
If onredirect is True then the timer is cancelled if redirect happens.
If onredirect is callable then in case of redirect the timer is
cancelled and this callable is called.
If onerror is True then the timer is cancelled if a render error
happens. If onerror is callable then in case of a render error the
timer is cancelled and this callable is called.
"""
timer = QTimer()
timer.setSingleShot(True)
timer_callback = functools.partial(
self._on_wait_timeout,
timer=timer,
callback=callback,
)
timer.timeout.connect(timer_callback)
self.logger.log("waiting %sms; timer %s" % (time_ms, id(timer)),
min_level=2)
timer.start(time_ms)
self._active_timers.add(timer)
if onredirect:
self._timers_to_cancel_on_redirect[timer] = onredirect
if onerror:
self._timers_to_cancel_on_error[timer] = onerror
def _on_wait_timeout(self, timer, callback):
self.logger.log("wait timeout for %s" % id(timer), min_level=2)
if timer in self._active_timers:
self._active_timers.remove(timer)
self._timers_to_cancel_on_redirect.pop(timer, None)
self._timers_to_cancel_on_error.pop(timer, None)
callback()
def _cancel_timer(self, timer, errback=None):
self.logger.log("cancelling timer %s" % id(timer), min_level=2)
if timer in self._active_timers:
self._active_timers.remove(timer)
timer.stop()
if callable(errback):
self.logger.log("calling timer errback", min_level=2)
errback()
timer.deleteLater()
def _cancel_timers(self, timers):
for timer, oncancel in list(timers.items()):
self._cancel_timer(timer, oncancel)
timers.pop(timer, None)
def _cancel_all_timers(self):
self.logger.log("cancelling %d remaining timers" %
len(self._active_timers),
min_level=2)
for timer in list(self._active_timers):
self._cancel_timer(timer)
def _on_url_changed(self, url):
# log history
url = unicode(url.toString())
cause_ev = self.web_page.har_log._prev_entry(url, -1)
if cause_ev:
self._history.append(without_private(cause_ev.data))
self._cancel_timers(self._timers_to_cancel_on_redirect)
def inject_js(self, filename):
"""
Load JS library from file ``filename`` to the current frame.
"""
# TODO: shouldn't it keep injected scripts after redirects/reloads?
with open(filename, 'rb') as f:
script = f.read().decode('utf-8')
return self.runjs(script)
def inject_js_libs(self, folder):
"""
Load all JS libraries from ``folder`` folder to the current frame.
"""
# TODO: shouldn't it keep injected scripts after redirects/reloads?
for jsfile in os.listdir(folder):
if jsfile.endswith('.js'):
filename = os.path.join(folder, jsfile)
self.inject_js(filename)
def runjs(self, js_source):
"""
Run JS code in page context and return the result.
Only string results are supported.
"""
frame = self.web_page.mainFrame()
res = frame.evaluateJavaScript(js_source)
return qt2py(res)
def store_har_timing(self, name):
self.web_page.har_log.store_timing(name)
def _jsconsole_enable(self):
# TODO: add public interface or make console available by default
if self._js_console is not None:
return
self._js_console = _JavascriptConsole()
frame = self.web_page.mainFrame()
frame.addToJavaScriptWindowObject('console', self._js_console)
def _jsconsole_messages(self):
# TODO: add public interface or make console available by default
if self._js_console is None:
return []
return self._js_console.messages[:]
def html(self):
""" Return HTML of the current main frame """
self.logger.log("getting HTML", min_level=2)
frame = self.web_page.mainFrame()
result = bytes(frame.toHtml().toUtf8())
self.store_har_timing("_onHtmlRendered")
return result
def png(self, width=None, height=None, b64=False):
""" Return screenshot in PNG format """
self.logger.log("getting PNG", min_level=2)
image = QImage(self.web_page.viewportSize(), QImage.Format_ARGB32)
painter = QPainter(image)
self.web_page.mainFrame().render(painter)
painter.end()
self.store_har_timing("_onScreenshotPrepared")
if width:
image = image.scaledToWidth(width, Qt.SmoothTransformation)
if height:
image = image.copy(0, 0, width, height)
b = QBuffer()
image.save(b, "png")
result = bytes(b.data())
if b64:
result = base64.b64encode(result)
self.store_har_timing("_onPngRendered")
return result
def iframes_info(self, children=True, html=True):
""" Return information about all iframes """
self.logger.log("getting iframes", min_level=3)
frame = self.web_page.mainFrame()
result = self._frame_to_dict(frame, children, html)
self.store_har_timing("_onIframesRendered")
return result
def har(self):
""" Return HAR information """
self.logger.log("getting HAR", min_level=3)
return self.web_page.har_log.todict()
def history(self):
""" Return history of 'main' HTTP requests """
self.logger.log("getting history", min_level=3)
return copy.deepcopy(self._history)
def last_http_status(self):
"""
Return HTTP status code of the currently loaded webpage
or None if it is not available.
"""
if not self._history:
return
try:
return self._history[-1]["response"]["status"]
except KeyError:
return
def _frame_to_dict(self, frame, children=True, html=True):
g = frame.geometry()
res = {
"url": unicode(frame.url().toString()),
"requestedUrl": unicode(frame.requestedUrl().toString()),
"geometry": (g.x(), g.y(), g.width(), g.height()),
"title": unicode(frame.title())
}
if html:
res["html"] = unicode(frame.toHtml())
if children:
res["childFrames"] = [
self._frame_to_dict(f, True, html)
for f in frame.childFrames()
]
res["frameName"] = unicode(frame.frameName())
return res
