def render_GET(self, request): #log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args)) request.starttime = time.time() render_options = RenderOptions.fromrequest(request, self.max_timeout) # check arguments before starting the render render_options.get_filters(self.pool) timeout = render_options.get_timeout() wait_time = render_options.get_wait() pool_d = self._get_render(request, render_options) timer = reactor.callLater(timeout + wait_time, pool_d.cancel) request.notifyFinish().addErrback(self._request_failed, pool_d, timer) pool_d.addCallback(self._cancel_timer, timer) pool_d.addCallback(self._write_output, request) pool_d.addErrback(self._on_timeout_error, request, timeout=timeout) pool_d.addErrback(self._on_render_error, request) pool_d.addErrback(self._on_bad_request, request) pool_d.addErrback(self._on_internal_error, request) pool_d.addBoth(self._finish_request, request, options=render_options.data) return NOT_DONE_YET
def open_tab(self, meta=None): if meta is None: meta = {} manager = network_manager.create_default() data = {} data['uid'] = id(data) self.factory[self].tab = PortiaBrowserTab( network_manager=manager, splash_proxy_factory=None, verbosity=0, render_options=RenderOptions(data, defaults.MAX_TIMEOUT), visible=True, ) main_frame = self.tab.web_page.mainFrame() cookiejar = PortiaCookieJar(self.tab.web_page, self) self.tab.web_page.cookiejar = cookiejar if meta.get('cookies'): cookiejar.put_client_cookies(meta['cookies']) if meta.get('storage'): self.storage = meta['storage'] main_frame.loadStarted.connect(self._on_load_started) self.js_api = PortiaJSApi(self) main_frame.javaScriptWindowObjectCleared.connect( self.populate_window_object) self.tab.set_images_enabled(False) self.tab.set_viewport(meta.get('viewport', _DEFAULT_VIEWPORT)) self.tab.set_user_agent(meta.get('user_agent', _DEFAULT_USER_AGENT)) self.tab.loaded = False
def open_tab(self, meta=None): if meta is None: meta = {} manager = PortiaNetworkManager(request_middlewares=[], response_middlewares=[], verbosity=defaults.VERBOSITY) manager.setCache(None) data = {} data['uid'] = id(data) self.factory[self].tab = BrowserTab( network_manager=manager, splash_proxy_factory=None, verbosity=0, render_options=RenderOptions(data, defaults.MAX_TIMEOUT), visible=True, ) manager.tab = self.tab main_frame = self.tab.web_page.mainFrame() cookiejar = PortiaCookieJar(self.tab.web_page, self) manager.cookiejar = cookiejar manager.setCookieJar(cookiejar) if meta.get('cookies'): cookiejar.put_client_cookies(meta['cookies']) main_frame.loadStarted.connect(self._on_load_started) self.js_api = PortiaJSApi(self) main_frame.javaScriptWindowObjectCleared.connect( self.populate_window_object) self.tab.set_images_enabled(True) self.tab.set_viewport(meta.get('viewport') or _DEFAULT_VIEWPORT) self.tab.set_user_agent(meta.get('user_agent') or _DEFAULT_USER_AGENT) self.tab.loaded = False
def _validate_params(self, request): options = RenderOptions.fromrequest(request, self.max_timeout) options.get_filters(self.pool) # check params = options.get_common_params(self.pool.js_profiles_path) params.update({"timeout": options.get_timeout(), "har": 1, "png": 1, "html": 1}) if self.lua_enabled: params.update({"lua_source": options.get_lua_source()}) return params
def _validate_params(self, request): options = RenderOptions.fromrequest(request, self.max_timeout) options.get_filters(self.pool) # check params = options.get_common_params(self.pool.js_profiles_path) params.update({ 'timeout': options.get_timeout(), 'har': 1, 'png': 1, 'html': 1, }) if self.lua_enabled: params.update({ 'lua_source': options.get_lua_source(), }) return params
def render_GET(self, request): #log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args)) request.starttime = time.time() render_options = RenderOptions.fromrequest(request, self.max_timeout) # process argument cache original_options = render_options.data.copy() expired_args = render_options.get_expired_args(self.argument_cache) if expired_args: error = self._write_expired_args(request, expired_args) self._log_stats(request, original_options, error) return b"\n" saved_args = render_options.save_args_to_cache(self.argument_cache) if saved_args: value = ';'.join("{}={}".format(name, value) for name, value in saved_args) request.setHeader(b'X-Splash-Saved-Arguments', value.encode('utf8')) render_options.load_cached_args(self.argument_cache) # check arguments before starting the render render_options.get_filters(self.pool) render_options.get_engine( browser_engines_enabled=self.browser_engines_enabled) timeout = render_options.get_timeout() pool_d = self._get_render(request, render_options) timer = reactor.callLater(timeout, pool_d.cancel) request.notifyFinish().addErrback(self._request_failed, pool_d, timer) pool_d.addCallback(self._cancel_timer, timer) pool_d.addCallback(self._write_output, request) pool_d.addErrback(self._on_timeout_error, request, timeout=timeout, timer=timer) pool_d.addErrback(self._on_render_error, request) pool_d.addErrback(self._on_bad_request, request) pool_d.addErrback(self._on_internal_error, request) pool_d.addErrback(self._on_unhandled_internal_error, request) pool_d.addBoth(self._finish_request, request, options=original_options) return NOT_DONE_YET
def open_tab(self, meta=None): if meta is None: meta = {} manager = SplashQNetworkAccessManager( request_middlewares=[], response_middlewares=[], verbosity=defaults.VERBOSITY ) manager.setCache(None) data = {} data['uid'] = id(data) self.factory[self].tab = PortiaBrowserTab( network_manager=manager, splash_proxy_factory=None, verbosity=defaults.VERBOSITY, render_options=RenderOptions(data, defaults.MAX_TIMEOUT), visible=True, ) manager.tab = self.tab self.tab.register_callback('on_request', self._configure_requests) self.tab.register_callback('on_response', self._set_tab_html) main_frame = self.tab.web_page.mainFrame() cookiejar = PortiaCookieJar(self.tab.web_page, self) manager.cookiejar = cookiejar manager.setCookieJar(cookiejar) if meta.get('cookies'): cookiejar.put_client_cookies(meta['cookies']) main_frame.loadStarted.connect(self._on_load_started) main_frame.loadFinished.connect(self._on_load_finished) self.js_api = PortiaJSApi(self) main_frame.javaScriptWindowObjectCleared.connect( self._on_javascript_window_cleared) main_frame.initialLayoutCompleted.connect(self._on_layout_completed) self.tab.set_images_enabled(True) self.tab.set_viewport(meta.get('viewport') or _DEFAULT_VIEWPORT) self.tab.set_user_agent(meta.get('user_agent') or _DEFAULT_USER_AGENT) self.tab.loaded = False
def init_browser(network_manager_factory): # TODO: support the same command-line options as HTTP server. # from splash.server import start_logging # class opts(object): # logfile = "./kernel.log" # start_logging(opts) proxy_factory = None # TODO data = {} data['uid'] = id(data) tab = BrowserTab( network_manager=network_manager_factory(), splash_proxy_factory=proxy_factory, verbosity=2, # TODO render_options=RenderOptions(data, defaults.MAX_TIMEOUT), # TODO: timeout visible=True, ) return tab
def render_GET(self, request): #log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args)) request.starttime = time.time() render_options = RenderOptions.fromrequest(request, self.max_timeout) render_options.get_filters(self.pool) # check filters earlier pool_d = self._getRender(request, render_options) timeout = render_options.get_timeout() wait_time = render_options.get_wait() timer = reactor.callLater(timeout + wait_time, pool_d.cancel) pool_d.addCallback(self._cancelTimer, timer) pool_d.addCallback(self._writeOutput, request) pool_d.addErrback(self._timeoutError, request) pool_d.addErrback(self._renderError, request) pool_d.addErrback(self._badRequest, request) pool_d.addErrback(self._internalError, request) pool_d.addBoth(self._finishRequest, request) return NOT_DONE_YET
def render_GET(self, request): #log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args)) request.starttime = time.time() render_options = RenderOptions.fromrequest(request) render_options.get_filters(self.pool) # check filters earlier pool_d = self._getRender(request, render_options) timeout = render_options.get_timeout() wait_time = render_options.get_wait() timer = reactor.callLater(timeout+wait_time, pool_d.cancel) pool_d.addCallback(self._cancelTimer, timer) pool_d.addCallback(self._writeOutput, request) pool_d.addErrback(self._timeoutError, request) pool_d.addErrback(self._renderError, request) pool_d.addErrback(self._badRequest, request) pool_d.addErrback(self._internalError, request) pool_d.addBoth(self._finishRequest, request) return NOT_DONE_YET
def _validate_params(self, request): options = RenderOptions.fromrequest(request, self.max_timeout) options.get_filters(self.pool) # check params = options.get_common_params(self.pool.js_profiles_path) params.update({ 'save_args': options.get_save_args(), 'load_args': options.get_load_args(), 'timeout': options.get_timeout(), 'har': 1, 'png': 1, 'html': 1, }) filters = options.get('filters', default='') if filters: params['filters'] = filters if self.lua_enabled: params.update({ 'lua_source': options.get_lua_source(), }) return params
def render_GET(self, request): #log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args)) request.starttime = time.time() render_options = RenderOptions.fromrequest(request, self.max_timeout) # process argument cache original_options = render_options.data.copy() expired_args = render_options.get_expired_args(self.argument_cache) if expired_args: error = self._write_expired_args(request, expired_args) self._log_stats(request, original_options, error) return b"\n" saved_args = render_options.save_args_to_cache(self.argument_cache) if saved_args: value = ';'.join("{}={}".format(name, value) for name, value in saved_args) request.setHeader(b'X-Splash-Saved-Arguments', value.encode('utf8')) render_options.load_cached_args(self.argument_cache) # check arguments before starting the render render_options.get_filters(self.pool) timeout = render_options.get_timeout() wait_time = render_options.get_wait() pool_d = self._get_render(request, render_options) timer = reactor.callLater(timeout+wait_time, pool_d.cancel) request.notifyFinish().addErrback(self._request_failed, pool_d, timer) pool_d.addCallback(self._cancel_timer, timer) pool_d.addCallback(self._write_output, request) pool_d.addErrback(self._on_timeout_error, request, timeout=timeout) pool_d.addErrback(self._on_render_error, request) pool_d.addErrback(self._on_bad_request, request) pool_d.addErrback(self._on_internal_error, request) pool_d.addBoth(self._finish_request, request, options=original_options) return NOT_DONE_YET
def render_GET(self, request): #log.msg("%s %s %s %s" % (id(request), request.method, request.path, request.args)) request.starttime = time.time() render_options = RenderOptions.fromrequest(request, self.max_timeout) # check arguments before starting the render render_options.get_filters(self.pool) timeout = render_options.get_timeout() wait_time = render_options.get_wait() pool_d = self._get_render(request, render_options) timer = reactor.callLater(timeout+wait_time, pool_d.cancel) pool_d.addCallback(self._cancel_timer, timer) pool_d.addCallback(self._write_output, request, options=render_options.data) pool_d.addErrback(self._on_timeout_error, request, timeout=timeout) pool_d.addErrback(self._on_render_error, request) pool_d.addErrback(self._on_bad_request, request) pool_d.addErrback(self._on_internal_error, request) pool_d.addBoth(self._finish_request, request) return NOT_DONE_YET
def _raise_proxy_error(description, **kwargs): RenderOptions.raise_error("proxy", description, **kwargs)