def render_GET(self, request):
#log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args))
request.starttime = time.time()
render_options = RenderOptions.fromrequest(request, self.max_timeout)
# check arguments before starting the render
render_options.get_filters(self.pool)
timeout = render_options.get_timeout()
wait_time = render_options.get_wait()
pool_d = self._get_render(request, render_options)
timer = reactor.callLater(timeout + wait_time, pool_d.cancel)
request.notifyFinish().addErrback(self._request_failed, pool_d, timer)
pool_d.addCallback(self._cancel_timer, timer)
pool_d.addCallback(self._write_output, request)
pool_d.addErrback(self._on_timeout_error, request, timeout=timeout)
pool_d.addErrback(self._on_render_error, request)
pool_d.addErrback(self._on_bad_request, request)
pool_d.addErrback(self._on_internal_error, request)
pool_d.addBoth(self._finish_request,
request,
options=render_options.data)
return NOT_DONE_YET
def open_tab(self, meta=None):
if meta is None:
meta = {}
manager = network_manager.create_default()
data = {}
data['uid'] = id(data)
self.factory[self].tab = PortiaBrowserTab(
network_manager=manager,
splash_proxy_factory=None,
verbosity=0,
render_options=RenderOptions(data, defaults.MAX_TIMEOUT),
visible=True,
)
main_frame = self.tab.web_page.mainFrame()
cookiejar = PortiaCookieJar(self.tab.web_page, self)
self.tab.web_page.cookiejar = cookiejar
if meta.get('cookies'):
cookiejar.put_client_cookies(meta['cookies'])
if meta.get('storage'):
self.storage = meta['storage']
main_frame.loadStarted.connect(self._on_load_started)
self.js_api = PortiaJSApi(self)
main_frame.javaScriptWindowObjectCleared.connect(
self.populate_window_object)
self.tab.set_images_enabled(False)
self.tab.set_viewport(meta.get('viewport', _DEFAULT_VIEWPORT))
self.tab.set_user_agent(meta.get('user_agent', _DEFAULT_USER_AGENT))
self.tab.loaded = False
def open_tab(self, meta=None):
if meta is None:
meta = {}
manager = PortiaNetworkManager(request_middlewares=[],
response_middlewares=[],
verbosity=defaults.VERBOSITY)
manager.setCache(None)
data = {}
data['uid'] = id(data)
self.factory[self].tab = BrowserTab(
network_manager=manager,
splash_proxy_factory=None,
verbosity=0,
render_options=RenderOptions(data, defaults.MAX_TIMEOUT),
visible=True,
)
manager.tab = self.tab
main_frame = self.tab.web_page.mainFrame()
cookiejar = PortiaCookieJar(self.tab.web_page, self)
manager.cookiejar = cookiejar
manager.setCookieJar(cookiejar)
if meta.get('cookies'):
cookiejar.put_client_cookies(meta['cookies'])
main_frame.loadStarted.connect(self._on_load_started)
self.js_api = PortiaJSApi(self)
main_frame.javaScriptWindowObjectCleared.connect(
self.populate_window_object)
self.tab.set_images_enabled(True)
self.tab.set_viewport(meta.get('viewport') or _DEFAULT_VIEWPORT)
self.tab.set_user_agent(meta.get('user_agent') or _DEFAULT_USER_AGENT)
self.tab.loaded = False
def _validate_params(self, request):
options = RenderOptions.fromrequest(request, self.max_timeout)
options.get_filters(self.pool) # check
params = options.get_common_params(self.pool.js_profiles_path)
params.update({"timeout": options.get_timeout(), "har": 1, "png": 1, "html": 1})
if self.lua_enabled:
params.update({"lua_source": options.get_lua_source()})
return params
def _validate_params(self, request):
options = RenderOptions.fromrequest(request, self.max_timeout)
options.get_filters(self.pool) # check
params = options.get_common_params(self.pool.js_profiles_path)
params.update({
'timeout': options.get_timeout(),
'har': 1,
'png': 1,
'html': 1,
})
if self.lua_enabled:
params.update({
'lua_source': options.get_lua_source(),
})
return params
def _validate_params(self, request):
options = RenderOptions.fromrequest(request, self.max_timeout)
options.get_filters(self.pool) # check
params = options.get_common_params(self.pool.js_profiles_path)
params.update({
'timeout': options.get_timeout(),
'har': 1,
'png': 1,
'html': 1,
})
if self.lua_enabled:
params.update({
'lua_source': options.get_lua_source(),
})
return params
def render_GET(self, request):
#log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args))
request.starttime = time.time()
render_options = RenderOptions.fromrequest(request, self.max_timeout)
# process argument cache
original_options = render_options.data.copy()
expired_args = render_options.get_expired_args(self.argument_cache)
if expired_args:
error = self._write_expired_args(request, expired_args)
self._log_stats(request, original_options, error)
return b"\n"
saved_args = render_options.save_args_to_cache(self.argument_cache)
if saved_args:
value = ';'.join("{}={}".format(name, value)
for name, value in saved_args)
request.setHeader(b'X-Splash-Saved-Arguments',
value.encode('utf8'))
render_options.load_cached_args(self.argument_cache)
# check arguments before starting the render
render_options.get_filters(self.pool)
render_options.get_engine(
browser_engines_enabled=self.browser_engines_enabled)
timeout = render_options.get_timeout()
pool_d = self._get_render(request, render_options)
timer = reactor.callLater(timeout, pool_d.cancel)
request.notifyFinish().addErrback(self._request_failed, pool_d, timer)
pool_d.addCallback(self._cancel_timer, timer)
pool_d.addCallback(self._write_output, request)
pool_d.addErrback(self._on_timeout_error,
request,
timeout=timeout,
timer=timer)
pool_d.addErrback(self._on_render_error, request)
pool_d.addErrback(self._on_bad_request, request)
pool_d.addErrback(self._on_internal_error, request)
pool_d.addErrback(self._on_unhandled_internal_error, request)
pool_d.addBoth(self._finish_request, request, options=original_options)
return NOT_DONE_YET
def open_tab(self, meta=None):
if meta is None:
meta = {}
manager = SplashQNetworkAccessManager(
request_middlewares=[],
response_middlewares=[],
verbosity=defaults.VERBOSITY
)
manager.setCache(None)
data = {}
data['uid'] = id(data)
self.factory[self].tab = PortiaBrowserTab(
network_manager=manager,
splash_proxy_factory=None,
verbosity=defaults.VERBOSITY,
render_options=RenderOptions(data, defaults.MAX_TIMEOUT),
visible=True,
)
manager.tab = self.tab
self.tab.register_callback('on_request', self._configure_requests)
self.tab.register_callback('on_response', self._set_tab_html)
main_frame = self.tab.web_page.mainFrame()
cookiejar = PortiaCookieJar(self.tab.web_page, self)
manager.cookiejar = cookiejar
manager.setCookieJar(cookiejar)
if meta.get('cookies'):
cookiejar.put_client_cookies(meta['cookies'])
main_frame.loadStarted.connect(self._on_load_started)
main_frame.loadFinished.connect(self._on_load_finished)
self.js_api = PortiaJSApi(self)
main_frame.javaScriptWindowObjectCleared.connect(
self._on_javascript_window_cleared)
main_frame.initialLayoutCompleted.connect(self._on_layout_completed)
self.tab.set_images_enabled(True)
self.tab.set_viewport(meta.get('viewport') or _DEFAULT_VIEWPORT)
self.tab.set_user_agent(meta.get('user_agent') or _DEFAULT_USER_AGENT)
self.tab.loaded = False
def init_browser(network_manager_factory):
# TODO: support the same command-line options as HTTP server.
# from splash.server import start_logging
# class opts(object):
# logfile = "./kernel.log"
# start_logging(opts)
proxy_factory = None # TODO
data = {}
data['uid'] = id(data)
tab = BrowserTab(
network_manager=network_manager_factory(),
splash_proxy_factory=proxy_factory,
verbosity=2, # TODO
render_options=RenderOptions(data, defaults.MAX_TIMEOUT), # TODO: timeout
visible=True,
)
return tab
def render_GET(self, request):
#log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args))
request.starttime = time.time()
render_options = RenderOptions.fromrequest(request, self.max_timeout)
render_options.get_filters(self.pool) # check filters earlier
pool_d = self._getRender(request, render_options)
timeout = render_options.get_timeout()
wait_time = render_options.get_wait()
timer = reactor.callLater(timeout + wait_time, pool_d.cancel)
pool_d.addCallback(self._cancelTimer, timer)
pool_d.addCallback(self._writeOutput, request)
pool_d.addErrback(self._timeoutError, request)
pool_d.addErrback(self._renderError, request)
pool_d.addErrback(self._badRequest, request)
pool_d.addErrback(self._internalError, request)
pool_d.addBoth(self._finishRequest, request)
return NOT_DONE_YET
def render_GET(self, request):
#log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args))
request.starttime = time.time()
render_options = RenderOptions.fromrequest(request)
render_options.get_filters(self.pool) # check filters earlier
pool_d = self._getRender(request, render_options)
timeout = render_options.get_timeout()
wait_time = render_options.get_wait()
timer = reactor.callLater(timeout+wait_time, pool_d.cancel)
pool_d.addCallback(self._cancelTimer, timer)
pool_d.addCallback(self._writeOutput, request)
pool_d.addErrback(self._timeoutError, request)
pool_d.addErrback(self._renderError, request)
pool_d.addErrback(self._badRequest, request)
pool_d.addErrback(self._internalError, request)
pool_d.addBoth(self._finishRequest, request)
return NOT_DONE_YET
def _validate_params(self, request):
options = RenderOptions.fromrequest(request, self.max_timeout)
options.get_filters(self.pool) # check
params = options.get_common_params(self.pool.js_profiles_path)
params.update({
'save_args': options.get_save_args(),
'load_args': options.get_load_args(),
'timeout': options.get_timeout(),
'har': 1,
'png': 1,
'html': 1,
})
filters = options.get('filters', default='')
if filters:
params['filters'] = filters
if self.lua_enabled:
params.update({
'lua_source': options.get_lua_source(),
})
return params
def _validate_params(self, request):
options = RenderOptions.fromrequest(request, self.max_timeout)
options.get_filters(self.pool) # check
params = options.get_common_params(self.pool.js_profiles_path)
params.update({
'save_args': options.get_save_args(),
'load_args': options.get_load_args(),
'timeout': options.get_timeout(),
'har': 1,
'png': 1,
'html': 1,
})
filters = options.get('filters', default='')
if filters:
params['filters'] = filters
if self.lua_enabled:
params.update({
'lua_source': options.get_lua_source(),
})
return params
def render_GET(self, request):
#log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args))
request.starttime = time.time()
render_options = RenderOptions.fromrequest(request, self.max_timeout)
# process argument cache
original_options = render_options.data.copy()
expired_args = render_options.get_expired_args(self.argument_cache)
if expired_args:
error = self._write_expired_args(request, expired_args)
self._log_stats(request, original_options, error)
return b"\n"
saved_args = render_options.save_args_to_cache(self.argument_cache)
if saved_args:
value = ';'.join("{}={}".format(name, value)
for name, value in saved_args)
request.setHeader(b'X-Splash-Saved-Arguments', value.encode('utf8'))
render_options.load_cached_args(self.argument_cache)
# check arguments before starting the render
render_options.get_filters(self.pool)
timeout = render_options.get_timeout()
wait_time = render_options.get_wait()
pool_d = self._get_render(request, render_options)
timer = reactor.callLater(timeout+wait_time, pool_d.cancel)
request.notifyFinish().addErrback(self._request_failed, pool_d, timer)
pool_d.addCallback(self._cancel_timer, timer)
pool_d.addCallback(self._write_output, request)
pool_d.addErrback(self._on_timeout_error, request, timeout=timeout)
pool_d.addErrback(self._on_render_error, request)
pool_d.addErrback(self._on_bad_request, request)
pool_d.addErrback(self._on_internal_error, request)
pool_d.addBoth(self._finish_request, request,
options=original_options)
return NOT_DONE_YET
def render_GET(self, request):
#log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args))
request.starttime = time.time()
render_options = RenderOptions.fromrequest(request, self.max_timeout)
# check arguments before starting the render
render_options.get_filters(self.pool)
timeout = render_options.get_timeout()
wait_time = render_options.get_wait()
pool_d = self._get_render(request, render_options)
timer = reactor.callLater(timeout+wait_time, pool_d.cancel)
pool_d.addCallback(self._cancel_timer, timer)
pool_d.addCallback(self._write_output, request, options=render_options.data)
pool_d.addErrback(self._on_timeout_error, request, timeout=timeout)
pool_d.addErrback(self._on_render_error, request)
pool_d.addErrback(self._on_bad_request, request)
pool_d.addErrback(self._on_internal_error, request)
pool_d.addBoth(self._finish_request, request)
return NOT_DONE_YET
def _raise_proxy_error(description, **kwargs):
RenderOptions.raise_error("proxy", description, **kwargs)
def _raise_proxy_error(description, **kwargs):
RenderOptions.raise_error("proxy", description, **kwargs)
