def test_resource_url_doesnt_proxy_non_http_or_https_urls_by_default(self): schemes = ['file', 'ws'] for scheme in schemes: url = '%s://ckan.org' % scheme data_dict = set_resource_url(url) non_proxied_url = proxy.get_proxified_resource_url(data_dict) proxied_url = proxy.get_proxified_resource_url(data_dict, scheme) assert non_proxied_url == url, non_proxied_url assert proxied_url != url, proxied_url
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = on_same_domain(data_dict) if not data_dict['resource'].get('format'): data_dict['resource']['format'] = \ self._guess_format_from_extension(data_dict['resource']['url']) proxy_service_url = None if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) proxy_service_url = get_proxified_service_url(data_dict) else: proxy_url = data_dict['resource']['url'] proxy_service_url = '' gapi_key = config.get('ckanext.geoview.gapi_key') if isinstance(self, p.IResourcePreview): p.toolkit.c.resource['proxy_url'] = proxy_url p.toolkit.c.resource['proxy_service_url'] = proxy_service_url p.toolkit.c.resource['gapi_key'] = gapi_key return {'proxy_service_url': proxy_service_url, 'proxy_url': proxy_url, 'gapi_key': gapi_key}
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = datapreview.on_same_domain(data_dict) p.toolkit.c.gapi_key = h.config.get('ckanext.geoview.gapi.key') if p.toolkit.check_ckan_version('2.3'): proxy_url = proxy.get_proxified_resource_url(data_dict) else: proxy_url = data_dict['resource']['url'] if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) return {'proxy_service_url': json.dumps(get_proxified_service_url(data_dict)), 'proxy_url': json.dumps(proxy_url)}
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy if self.proxy_enabled and not data_dict["resource"]["on_same_domain"]: p.toolkit.c.resource["proxy_url"] = proxy.get_proxified_resource_url(data_dict) else: p.toolkit.c.resource["proxy_url"] = data_dict["resource"]["url"]
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = on_same_domain(data_dict) if not data_dict['resource'].get('format'): data_dict['resource']['format'] = \ self._guess_format_from_extension(data_dict['resource']['url']) if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) proxy_service_url = get_proxified_service_url(data_dict) else: proxy_url = data_dict['resource']['url'] proxy_service_url = data_dict['resource']['url'] gapi_key = config.get('ckanext.geoview.gapi_key') if not p.toolkit.check_ckan_version(min_version='2.3'): p.toolkit.c.resource['proxy_url'] = proxy_url p.toolkit.c.resource['proxy_service_url'] = proxy_service_url p.toolkit.c.resource['gapi_key'] = gapi_key return {'resource_view_json': 'resource_view' in data_dict and json.dumps(data_dict['resource_view']), 'proxy_service_url': proxy_service_url, 'proxy_url': proxy_url, 'gapi_key': gapi_key, 'basemapsConfig' : self.basemapsConfig}
def test_invalid_url(self): self.data_dict = set_resource_url('javascript:downloadFile(foo)') proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 409, result.status assert 'Invalid URL' in result.body, result.body
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = on_same_domain(data_dict) if not data_dict["resource"].get("format"): data_dict["resource"][ "format"] = self._guess_format_from_extension( data_dict["resource"]["url"]) if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) proxy_service_url = utils.get_proxified_service_url(data_dict) else: proxy_url = data_dict["resource"]["url"] proxy_service_url = data_dict["resource"]["url"] gapi_key = toolkit.config.get("ckanext.geoview.gapi_key") return { "resource_view_json": "resource_view" in data_dict and json.dumps(data_dict["resource_view"]), "proxy_service_url": proxy_service_url, "proxy_url": proxy_url, "gapi_key": gapi_key, "basemapsConfig": self.basemapsConfig, }
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy if self.proxy_enabled and not data_dict['resource']['on_same_domain']: p.toolkit.c.resource[ 'proxy_url'] = proxy.get_proxified_resource_url(data_dict) else: p.toolkit.c.resource['proxy_url'] = data_dict['resource']['url']
def test_large_file(self): self.set_resource_url('http://0.0.0.0:50001/huge.json') proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 500, result.status assert 'too large' in result.body, result.body
def test_invalid_url(self): self.data_dict = set_resource_url('http:invalid_url') proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 409, result.status assert 'Invalid URL' in result.body, result.body
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy if (self.proxy_enabled and not data_dict['resource']['on_same_domain']): p.toolkit.c.resource['original_url'] = p.toolkit.c.resource['url'] p.toolkit.c.resource['url'] = proxy.get_proxified_resource_url( data_dict)
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = on_same_domain(data_dict) if not data_dict['resource'].get('format'): data_dict['resource']['format'] = \ self._guess_format_from_extension(data_dict['resource']['url']) if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) proxy_service_url = get_proxified_service_url(data_dict) else: proxy_url = data_dict['resource']['url'] proxy_service_url = data_dict['resource']['url'] gapi_key = toolkit.config.get('ckanext.geoview.gapi_key') if not toolkit.check_ckan_version(min_version='2.3'): toolkit.c.resource['proxy_url'] = proxy_url toolkit.c.resource['proxy_service_url'] = proxy_service_url toolkit.c.resource['gapi_key'] = gapi_key return { 'resource_view_json': 'resource_view' in data_dict and json.dumps(data_dict['resource_view']), 'proxy_service_url': proxy_service_url, 'proxy_url': proxy_url, 'gapi_key': gapi_key, 'basemapsConfig': self.basemapsConfig }
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = on_same_domain(data_dict) if not data_dict['resource'].get('format'): data_dict['resource']['format'] = \ self._guess_format_from_extension(data_dict['resource']['url']) proxy_service_url = None if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) proxy_service_url = get_proxified_service_url(data_dict) else: proxy_url = data_dict['resource']['url'] proxy_service_url = '' gapi_key = config.get('ckanext.geoview.gapi_key') if isinstance(self, p.IResourcePreview): p.toolkit.c.resource['proxy_url'] = proxy_url p.toolkit.c.resource['proxy_service_url'] = proxy_service_url p.toolkit.c.resource['gapi_key'] = gapi_key return { 'proxy_service_url': proxy_service_url, 'proxy_url': proxy_url, 'gapi_key': gapi_key }
def test_large_file_streaming(self): cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls(self.url, stream=True, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status_int == 409, result.status assert 'too large' in result.body, result.body
def test_invalid_url(self, app): responses.add_passthru(config['solr_url']) self.data_dict = set_resource_url('http:invalid_url') proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = app.get(proxied_url) assert result.status_code == 409 assert six.b('Invalid URL') in result.data
def test_large_file_streaming(self, app, ckan_config): cl = ckan_config.get_value(u'ckan.resource_proxy.max_file_size') + 1 self.mock_out_urls(self.url, stream=True, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = app.get(proxied_url) assert result.status_code == 409 assert six.b('too large') in result.data
def test_invalid_url(self): responses.add_passthru(config['solr_url']) self.data_dict = set_resource_url('http:invalid_url') proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status_int == 409, result.status assert 'Invalid URL' in result.body, result.body
def test_large_file_streaming(self, app): cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls(self.url, stream=True, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = app.get(proxied_url) assert result.status_code == 409 assert six.b('too large') in result.data
def test_large_file(self): cl = controller.MAX_FILE_SIZE + 1 self.register(self.url, content_length=cl, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status_int == 409, result.status_int assert 'too large' in result.body, result.body
def test_large_file_streaming(self): cl = controller.MAX_FILE_SIZE + 1 self.register(self.url, streaming=True, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 409, result.status assert 'too large' in result.body, result.body
def test_non_existent_resource(self): self.data_dict = {'package': {'name': 'doesnotexist'}, 'resource': {'id': 'doesnotexist'}} proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 404, result.status assert 'Resource not found' in result.body, result.body
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy self.same_domain = data_dict['resource'].get('on_same_domain') if self.proxy_is_enabled and not self.same_domain: data_dict['resource']['proxy_url'] = proxy.get_proxified_resource_url(data_dict) else: data_dict['resource']['proxy_url'] = data_dict['resource']['url']
def test_proxied_resource_url_proxies_http_and_https_by_default(self): http_url = 'http://ckan.org' https_url = 'https://ckan.org' for url in [http_url, https_url]: data_dict = set_resource_url(url) proxied_url = proxy.get_proxified_resource_url(data_dict) assert proxied_url != url, proxied_url
def setup_template_variables(self, context, data_dict): assert self.can_preview(data_dict) resource = data_dict['resource'] format_lower = resource['format'].lower() if format_lower in self.JSON_FORMATS and self.proxy_is_enabled and not resource[ 'on_same_domain']: p.toolkit.c.resource['url'] = proxy.get_proxified_resource_url( data_dict)
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy self.same_domain = data_dict['resource'].get('on_same_domain') if self.proxy_enabled and not self.same_domain: data_dict['resource']['original_url'] = \ data_dict['resource'].get('url') data_dict['resource']['url'] = \ proxy.get_proxified_resource_url(data_dict)
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy self.same_domain = data_dict["resource"].get("on_same_domain") if self.proxy_enabled and not self.same_domain: data_dict["resource"]["original_url"] = data_dict["resource"].get( "url") data_dict["resource"]["url"] = proxy.get_proxified_resource_url( data_dict)
def setup_template_variables(self, context, data_dict): if (self.proxy_is_enabled and not data_dict['resource']['on_same_domain']): url = proxy.get_proxified_resource_url(data_dict) p.toolkit.c.resource['url'] = url iframe_src = "{}?{}".format( self.arcgis_webmap_viewer_url_embed, urllib.urlencode(self.webmap_viewer_embed_params)) p.toolkit.c.iframe_src = iframe_src
def test_large_file(self, app, ckan_config): cl = ckan_config.get_value(u'ckan.resource_proxy.max_file_size') + 1 self.mock_out_urls(self.url, headers={'Content-Length': str(cl)}, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = app.get(proxied_url) assert result.status_code == 409 assert six.b('too large') in result.data
def test_resource_url_doesnt_proxy_non_http_or_https_urls_by_default(self): schemes = ['file', 'ws'] for scheme in schemes: url = '%s://ckan.org' % scheme self.resource['url'] = url non_proxied_url = proxied_url = proxy.get_proxified_resource_url({ 'package': self.dataset, 'resource': self.resource }) proxied_url = proxied_url = proxy.get_proxified_resource_url({ 'package': self.dataset, 'resource': self.resource }, scheme ) assert non_proxied_url == url, non_proxied_url assert proxied_url != url, proxied_url
def test_resource_proxy_on_404(self): self.set_resource_url('http://0.0.0.0:50001/foo.bar') url = self.data_dict['resource']['url'] result = requests.get(url) assert result.status_code == 404, result.status_code proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 404, result.status
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy p.toolkit.c.gapi_key = h.config.get('ckanext.geoview.gapi.key') if self.proxy_enabled and not data_dict['resource']['on_same_domain']: p.toolkit.c.resource['proxy_url'] = proxy.get_proxified_resource_url(data_dict) p.toolkit.c.resource['proxy_service_url'] = get_proxified_service_url(data_dict) else: p.toolkit.c.resource['proxy_url'] = data_dict['resource']['url']
def test_large_file(self): cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls(self.url, headers={'Content-Length': six.text_type(cl)}, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status_int == 409, result.status assert 'too large' in result.body, result.body
def test_large_file(self): httpretty.HTTPretty.register_uri( httpretty.HTTPretty.GET, self.url, content_length=controller.MAX_FILE_SIZE + 1, body=JSON_STRING) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 500, result.status assert 'too large' in result.body, result.body
def test_large_file(self, app): cl = blueprint.MAX_FILE_SIZE + 1 self.mock_out_urls(self.url, headers={'Content-Length': str(cl)}, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = app.get(proxied_url) assert result.status_code == 409 assert six.b('too large') in result.data
def test_proxied_resource_url_proxies_http_and_https_by_default(self): http_url = 'http://ckan.org' https_url = 'https://ckan.org' for url in [http_url, https_url]: self.resource['url'] = url proxied_url = proxy.get_proxified_resource_url({ 'package': self.dataset, 'resource': self.resource }) assert proxied_url != url, proxied_url
def test_large_file(self): cl = controller.MAX_FILE_SIZE + 1 self.mock_out_urls( self.url, headers={'Content-Length': six.text_type(cl)}, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status_int == 409, result.status assert 'too large' in result.body, result.body
def test_large_file_streaming(self): cl = controller.MAX_FILE_SIZE + 1 self.register( self.url, streaming=True, body='c' * cl) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 409, result.status assert 'too large' in result.body, result.body
def test_resource_proxy_non_existent(self): self.data_dict = set_resource_url('http://foo.bar') def f1(): url = self.data_dict['resource']['url'] requests.get(url) self.assertRaises(requests.ConnectionError, f1) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 500, result.status assert 'connection error' in result.body, result.body
def test_resource_proxy_on_200(self): self.set_resource_url('http://0.0.0.0:50001/test.json') url = self.data_dict['resource']['url'] result = requests.get(url) assert result.status_code == 200, result.status_code assert "yes, I'm proxied" in result.content, result.content proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 200, result.status assert "yes, I'm proxied" in result.body, result.body
def test_non_existent_url(self): self.data_dict = set_resource_url('http://foo.bar') def f1(): url = self.data_dict['resource']['url'] requests.get(url) self.assertRaises(requests.ConnectionError, f1) proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 502, result.status assert 'connection error' in result.body, result.body
def setup_template_variables(self, context, data_dict): # metadata = {'text_formats': self.text_formats, # 'json_formats': self.json_formats, # 'jsonp_formats': self.jsonp_formats, # 'xml_formats': self.xml_formats} url = proxy.get_proxified_resource_url(data_dict) return { # 'preview_metadata': json.dumps(metadata), 'resource_json': json.dumps(data_dict['resource']), 'resource_url': json.dumps(url) }
def test_resource_proxy_on_404(self): httpretty.HTTPretty.register_uri( httpretty.HTTPretty.GET, self.url, body="I'm not here", content_type='application/json', status=404) url = self.data_dict['resource']['url'] result = requests.get(url) assert result.status_code == 404, result.status_code proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 404, result.status
def setup_template_variables(self, context, data_dict): metadata = {'text_formats': self.text_formats, 'json_formats': self.json_formats, 'jsonp_formats': self.jsonp_formats, 'xml_formats': self.xml_formats} url = proxy.get_proxified_resource_url(data_dict) format_lower = data_dict['resource']['format'].lower() if format_lower in self.jsonp_formats: url = data_dict['resource']['url'] return {'preview_metadata': json.dumps(metadata), 'resource_json': json.dumps(data_dict['resource']), 'resource_url': json.dumps(url)}
def setup_template_variables(self, context, data_dict): assert self.can_preview(data_dict) p.toolkit.c.preview_metadata = json.dumps( { "text_formats": self.text_formats, "json_formats": self.json_formats, "jsonp_formats": self.jsonp_formats, "xml_formats": self.xml_formats, } ) resource = data_dict["resource"] format_lower = resource["format"].lower() if format_lower in self.no_jsonp_formats and self.proxy_is_enabled and not resource["on_same_domain"]: p.toolkit.c.resource["url"] = proxy.get_proxified_resource_url(data_dict)
def setup_template_variables(self, context, data_dict): assert self.can_preview(data_dict) p.toolkit.c.preview_metadata = json.dumps({ 'text_formats': self.text_formats, 'json_formats': self.json_formats, 'jsonp_formats': self.jsonp_formats, 'xml_formats': self.xml_formats }) resource = data_dict['resource'] format_lower = resource['format'].lower() if (format_lower in self.no_jsonp_formats and self.proxy_is_enabled and not resource['on_same_domain']): url = proxy.get_proxified_resource_url(data_dict) p.toolkit.c.resource['url'] = url
def test_resource_proxy_on_404(self): self.register( self.url, body="I'm not here", content_type='application/json', status=404) url = self.data_dict['resource']['url'] result = requests.get(url) assert result.status_code == 404, result.status_code proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') # we expect a 409 because the resourceproxy got an error (404) # from the server assert result.status == 409, result.status assert '404' in result.body
def setup_template_variables(self, context, data_dict): metadata = { "text_formats": self.text_formats, "json_formats": self.json_formats, "jsonp_formats": self.jsonp_formats, "xml_formats": self.xml_formats, } url = proxy.get_proxified_resource_url(data_dict) format_lower = data_dict["resource"]["format"].lower() if format_lower in self.jsonp_formats: url = data_dict["resource"]["url"] return { "preview_metadata": json.dumps(metadata), "resource_json": json.dumps(data_dict["resource"]), "resource_url": json.dumps(url), }
def setup_template_variables(self, context, data_dict): assert self.can_preview(data_dict) resource = data_dict["resource"] format_lower = resource["format"].lower() if format_lower in self.JSON_FORMATS and self.proxy_is_enabled and not resource["on_same_domain"]: p.toolkit.c.resource["url"] = proxy.get_proxified_resource_url(data_dict)
def setup_template_variables(self, context, data_dict): proxified_url = proxy.get_proxified_resource_url(data_dict) return { 'proxied_url': json.dumps(proxified_url) }
def setup_template_variables(self, context, data_dict): if p.toolkit.check_ckan_version(min_version='2.3'): metadata = {'ngsi_formats': self.NGSI_FORMATS} resource = data_dict['resource'] proxy_enabled = p.plugin_loaded('resource_proxy') oauth2_enabled = p.plugin_loaded('oauth2') same_domain = datapreview.on_same_domain(data_dict) if 'oauth_req' not in resource: oauth_req = 'false' else: oauth_req = resource['oauth_req'] if proxy_enabled and not same_domain: if check_query(resource): if oauth_req == 'true' and not p.toolkit.c.user: details = "</br></br>In order to see this resource properly, you need to be logged in.</br></br></br>" f_details = "In order to see this resource properly, you need to be logged in." h.flash_error(f_details, allow_html=False) view_enable = [False, details] url = proxy.get_proxified_resource_url(data_dict) elif oauth_req == 'true' and not oauth2_enabled: details = "</br></br>In order to see this resource properly, enable oauth2 extension</br></br></br>" f_details = "In order to see this resource properly, enable oauth2 extension." h.flash_error(f_details, allow_html=False) view_enable = [False, details] url = proxy.get_proxified_resource_url(data_dict) else: url = self.get_proxified_ngsi_url(data_dict) data_dict['resource']['url'] = url view_enable = [True, 'OK'] else: details = "</br></br>This is not a ContextBroker query, please check <a href='https://forge.fiware.org/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide'>Orion Context Broker documentation</a></br></br></br>" f_details = "This is not a ContextBroker query, please check Orion Context Broker documentation." h.flash_error(f_details, allow_html=False) view_enable = [False, details] url = proxy.get_proxified_resource_url(data_dict) else: if proxy_enabled: details = "</br></br>Enable resource_proxy</br></br></br>" f_details = "Enable resource_proxy." else: details = "</br></br>This is not a ContextBroker query, please check <a href='https://forge.fiware.org/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide'>Orion Context Broker documentation</a></br></br></br>" f_details = "This is not a ContextBroker query, please check Orion Context Broker documentation." h.flash_error(f_details, allow_html=False) view_enable = [False, details] url = '' return {'preview_metadata': json.dumps(metadata), 'resource_json': json.dumps(data_dict['resource']), 'resource_url': json.dumps(url), 'view_enable': json.dumps(view_enable)} else: if self.proxy_is_enabled and not data_dict['resource']['on_same_domain']: if check_query(data_dict['resource']): url = self.get_proxified_ngsi_url(data_dict) p.toolkit.c.resource['url'] = url
def setup_template_variables(self, context, data_dict): if (self.proxy_is_enabled and not data_dict['resource']['on_same_domain']): url = proxy.get_proxified_resource_url(data_dict) p.toolkit.c.resource['url'] = url