def test_translation_works_on_flask_and_pylons(self):
app = helpers._get_test_app()
if not p.plugin_loaded(u'test_routing_plugin'):
p.load(u'test_routing_plugin')
try:
plugin = p.get_plugin(u'test_routing_plugin')
app.flask_app.register_extension_blueprint(
plugin.get_blueprint())
resp = app.get(u'/flask_translated')
eq_(resp.body, u'Dataset')
resp = app.get(u'/es/flask_translated')
eq_(resp.body, u'Conjunto de datos')
resp = app.get(u'/pylons_translated')
eq_(resp.body, u'Groups')
resp = app.get(u'/es/pylons_translated')
eq_(resp.body, u'Grupos')
finally:
if p.plugin_loaded(u'test_routing_plugin'):
p.unload(u'test_routing_plugin')
def setup_class(cls):
if not p.plugin_loaded("image_view"):
p.load("image_view")
if not p.plugin_loaded("recline_view"):
p.load("recline_view")
helpers.reset_db()
def setup_class(cls):
if not p.plugin_loaded('datastore'):
p.load('datastore')
if not p.plugin_loaded('datapusher'):
p.load('datapusher')
helpers.reset_db()
def setup_class(cls):
if not p.plugin_loaded('image_view'):
p.load('image_view')
if not p.plugin_loaded('recline_view'):
p.load('recline_view')
helpers.reset_db()
def setup_class(cls):
if not p.plugin_loaded("datastore"):
p.load("datastore")
if not p.plugin_loaded("datapusher"):
p.load("datapusher")
helpers.reset_db()
def setup_class(cls):
if not p.plugin_loaded('image_view'):
p.load('image_view')
if not p.plugin_loaded('recline_view'):
p.load('recline_view')
if not p.plugin_loaded('webpage_view'):
p.load('webpage_view')
if not p.plugin_loaded('test_datastore_view'):
p.load('test_datastore_view')
def setup_class(cls):
super(TestChartView, cls).setup_class()
if not p.plugin_loaded('c3charts'):
p.load('c3charts')
if not p.plugin_loaded('datastore'):
p.load('datastore')
def setup_class(cls):
cls.app = _get_test_app()
if not p.plugin_loaded('recline_view'):
p.load('recline_view')
if not p.plugin_loaded('datastore'):
p.load('datastore')
app_config = config.copy()
app_config['ckan.legacy_templates'] = 'false'
app_config['ckan.plugins'] = 'recline_view datastore'
app_config['ckan.views.default_views'] = 'recline_view'
def setup_class(cls):
cls.app = helpers._get_test_app()
if not p.plugin_loaded('datastore'):
p.load('datastore')
if not p.plugin_loaded('datapusher'):
p.load('datapusher')
helpers.reset_db()
def setup_class(cls):
if not is_datastore_supported():
raise nose.SkipTest('Datastore not supported')
if not p.plugin_loaded('datastore'):
p.load('datastore')
if not p.plugin_loaded('datapusher'):
p.load('datapusher')
if not p.plugin_loaded('recline_grid_view'):
p.load('recline_grid_view')
helpers.reset_db()
def setup_class(cls):
if not p.plugin_loaded('recline_view'):
p.load('recline_view')
if not p.plugin_loaded('datastore'):
p.load('datastore')
app_config = config.copy()
app_config['ckan.legacy_templates'] = 'false'
app_config['ckan.plugins'] = 'recline_view datastore'
app_config['ckan.views.default_views'] = 'recline_view'
wsgiapp = middleware.make_app(config['global_conf'], **app_config)
cls.app = paste.fixture.TestApp(wsgiapp)
def setup_class(cls):
if not plugins.plugin_loaded('text_view'):
plugins.load('text_view')
if plugins.plugin_loaded('resource_proxy'):
plugins.unload('resource_proxy')
cls.p = plugin.TextView()
create_test_data.CreateTestData.create()
cls.resource_view, cls.package, cls.resource_id = \
_create_test_view(cls.view_type)
def test_api_info(self):
dataset = factories.Dataset()
resource = factories.Resource(
id='588dfa82-760c-45a2-b78a-e3bc314a4a9b',
package_id=dataset['id'], datastore_active=True)
# the 'API info' is seen on the resource_read page, a snippet loaded by
# javascript via data_api_button.html
url = template_helpers.url_for(
controller='api', action='snippet', ver=1,
snippet_path='api_info.html', resource_id=resource['id'])
if not p.plugin_loaded('datastore'):
p.load('datastore')
app = self._get_test_app()
page = app.get(url, status=200)
p.unload('datastore')
# check we built all the urls ok
expected_urls = (
'http://test.ckan.net/api/3/action/datastore_create',
'http://test.ckan.net/api/3/action/datastore_upsert',
'<code>http://test.ckan.net/api/3/action/datastore_search',
'http://test.ckan.net/api/3/action/datastore_search_sql',
'http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&limit=5',
'http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&q=jones',
'http://test.ckan.net/api/3/action/datastore_search_sql?sql=SELECT * from "588dfa82-760c-45a2-b78a-e3bc314a4a9b" WHERE title LIKE 'jones'',
"url: 'http://test.ckan.net/api/3/action/datastore_search'",
"http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&limit=5&q=title:jones",
)
for url in expected_urls:
assert url in page, url
def setup_class(cls):
cls._original_config = config.copy()
cls.app = _get_test_app()
if not p.plugin_loaded('resource_proxy'):
p.load('resource_proxy')
config['ckan.plugins'] = 'resource_proxy'
create_test_data.CreateTestData.create()
def test_ask_around_flask_core_and_pylons_extension_route(self):
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
app = self._get_test_app()
# We want our CKAN app, not the WebTest one
app = app.app
environ = {
'PATH_INFO': '/pylons_and_flask',
'REQUEST_METHOD': 'GET',
}
wsgiref.util.setup_testing_defaults(environ)
answers = app.ask_around('can_handle_request', environ)
answers = sorted(answers, key=lambda a: a[1])
eq_(len(answers), 2)
eq_([a[0] for a in answers], [True, True])
eq_([a[1] for a in answers], ['flask_app', 'pylons_app'])
# TODO: we still can't distinguish between Flask core and extension
# eq_(answers[0][2], 'extension')
eq_(answers[1][2], 'extension')
p.unload('test_routing_plugin')
def test_ask_around_pylons_extension_route_post_using_get(self):
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
app = self._get_test_app()
# We want our CKAN app, not the WebTest one
app = app.app
environ = {
'PATH_INFO': '/from_pylons_extension_before_map_post_only',
'REQUEST_METHOD': 'GET',
}
wsgiref.util.setup_testing_defaults(environ)
answers = app.ask_around('can_handle_request', environ)
# We are going to get an answer from Pylons, but just because it will
# match the catch-all template route, hence the `core` origin.
eq_(len(answers), 1)
eq_(answers[0][0], True)
eq_(answers[0][1], 'pylons_app')
eq_(answers[0][2], 'core')
p.unload('test_routing_plugin')
def teardown_class(cls):
if p.plugin_loaded('webpage_view'):
p.unload('webpage_view')
super(TestWebPageView, cls).teardown_class()
helpers.reset_db()
def setup_class(cls):
super(cls, cls).setup_class()
if not p.plugin_loaded('image_view'):
p.load('image_view')
helpers.reset_db()
def setup_class(cls):
harvest_model.setup()
CreateTestData.create()
sysadmin_user = ckan.model.User.get('testsysadmin')
cls.sysadmin = {
'id': sysadmin_user.id,
'apikey': sysadmin_user.apikey,
'name': sysadmin_user.name,
}
cls.app = paste.fixture.TestApp(pylons.test.pylonsapp)
cls.default_source_dict = {
"url": "http://test.action.com",
"name": "test-source-action",
"title": "Test source action",
"notes": "Test source action desc",
"source_type": "test-for-action",
"frequency": "MANUAL",
"config": json.dumps({"custom_option":["a","b"]})
}
if not p.plugin_loaded('test_action_harvester'):
p.load('test_action_harvester')
def teardown_class(cls):
if plugins.plugin_loaded('example_idatasetform'):
plugins.unload('example_idatasetform')
helpers.reset_db()
ckan.lib.search.clear_all()
config.clear()
config.update(cls.original_config)
def setup(self):
self.app = helpers._get_test_app()
# Install plugin and register its blueprint
if not plugins.plugin_loaded(u'example_flask_iblueprint'):
plugins.load(u'example_flask_iblueprint')
plugin = plugins.get_plugin(u'example_flask_iblueprint')
self.app.flask_app.register_extension_blueprint(plugin.get_blueprint())
def teardown_class(cls):
config.clear()
config.update(cls._original_config)
model.repo.rebuild_db()
# Reenable Solr indexing
if (sys.version_info[0] == 2 and sys.version_info[1] == 6
and not p.plugin_loaded('synchronous_search')):
p.load('synchronous_search')
def teardown_class(cls):
rebuild_all_dbs(cls.Session)
p.unload('datastore')
p.unload('datapusher')
# Reenable Solr indexing
if (sys.version_info[0] == 2 and sys.version_info[1] == 6
and not p.plugin_loaded('synchronous_search')):
p.load('synchronous_search')
def _inject_views_into_page(self, _page):
# this is a good proxy to a version of CKAN with views enabled.
if not p.plugin_loaded('image_view'):
return
try:
import lxml
import lxml.html
except ImportError:
return
try:
root = lxml.html.fromstring(_page['content'])
# Return if any errors are found while parsing the content
except (lxml.etree.XMLSyntaxError,
lxml.etree.ParserError):
return
for element in root.findall('.//iframe'):
embed_element = element.attrib.pop('data-ckan-view-embed', None)
if not embed_element:
continue
element.tag = 'div'
error = None
try:
iframe_src = element.attrib.pop('src', '')
width = element.attrib.pop('width', '80')
if not width.endswith('%') and not width.endswith('px'):
width = width + 'px'
height = element.attrib.pop('height', '80')
if not height.endswith('%') and not height.endswith('px'):
height = height + 'px'
align = element.attrib.pop('align', 'none')
style = "width: %s; height: %s; float: %s; overflow: auto; vertical-align:middle; position:relative" % (width, height, align)
element.attrib['style'] = style
element.attrib['class'] = 'pages-embed'
view = p.toolkit.get_action('resource_view_show')({}, {'id': iframe_src[-36:]})
context = {}
resource = p.toolkit.get_action('resource_show')(context, {'id': view['resource_id']})
package_id = context['resource'].resource_group.package_id
package = p.toolkit.get_action('package_show')(context, {'id': package_id})
except p.toolkit.ObjectNotFound:
error = _('ERROR: View not found {view_id}'.format(view_id=iframe_src ))
if error:
resource_view_html = '<h4> %s </h4>' % error
elif not helpers.resource_view_is_iframed(view):
resource_view_html = helpers.rendered_resource_view(view, resource, package)
else:
src = helpers.url_for(qualified=True, controller='package', action='resource_view', id=package['name'], resource_id=resource['id'], view_id=view['id'])
message = _('Your browser does not support iframes.')
resource_view_html = '<iframe src="{src}" frameborder="0" width="100%" height="100%" style="display:block"> <p>{message}</p> </iframe>'.format(src=src, message=message)
view_element = lxml.html.fromstring(resource_view_html)
element.append(view_element)
_page['content'] = lxml.html.tostring(root)
def test_resource_view_factory(self):
if not p.plugin_loaded("image_view"):
p.load("image_view")
resource_view1 = factories.ResourceView()
resource_view2 = factories.ResourceView()
assert_not_equals(resource_view1["id"], resource_view2["id"])
p.unload("image_view")
def test_resource_view_factory(self):
if not p.plugin_loaded('image_view'):
p.load('image_view')
resource_view1 = factories.ResourceView()
resource_view2 = factories.ResourceView()
assert_not_equals(resource_view1['id'], resource_view2['id'])
p.unload('image_view')
def setup(self):
self.app = helpers._get_test_app()
# Install plugin and register its blueprint
if not p.plugin_loaded(u'test_flash_plugin'):
p.load(u'test_flash_plugin')
plugin = p.get_plugin(u'test_flash_plugin')
self.app.flask_app.register_extension_blueprint(
plugin.get_blueprint())
def test_helper_exception_non_existing_helper_as_item(self):
'''Calling a non-existing helper on `h` raises a HelperException.'''
if not p.plugin_loaded('test_helpers_plugin'):
p.load('test_helpers_plugin')
app = self._get_test_app()
app.get('/broken_helper_as_item')
p.unload('test_helpers_plugin')
def setup_class(cls):
super(TestFlaskUserIdentifiedInRequest, cls).setup_class()
cls.app = cls._get_test_app()
cls.flask_app = cls.app.flask_app
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
plugin = p.get_plugin('test_routing_plugin')
cls.flask_app.register_extension_blueprint(
plugin.get_blueprint())
def can_view(self, data_dict):
resource = data_dict['resource']
format_lower = resource['format'].lower()
proxy_enabled = p.plugin_loaded('resource_proxy')
same_domain = datapreview.on_same_domain(data_dict)
if format_lower in self.jsonp_formats:
return True
if format_lower in self.no_jsonp_formats:
return proxy_enabled or same_domain
return False
def test_ask_around_flask_core_and_pylons_extension_route(self):
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
app = self._get_test_app()
# We want our CKAN app, not the WebTest one
app = app.app
environ = {
'PATH_INFO': '/pylons_and_flask',
'REQUEST_METHOD': 'GET',
}
wsgiref.util.setup_testing_defaults(environ)
answers = app.ask_around(environ)
answers = sorted(answers, key=lambda a: a[1])
eq_(answers, [(True, 'flask_app'), (True, 'pylons_app', 'extension')])
p.unload('test_routing_plugin')
def get_proxified_resource_url(
data_dict: dict[str, Any],
proxy_schemes: Container[str] = ("http", "https"),
):
"""
:param data_dict: contains a resource and package dict
:type data_dict: dictionary
:param proxy_schemes: list of url schemes to proxy for.
:type data_dict: list
"""
url = data_dict[u'resource'][u'url']
if not p.plugin_loaded(u'resource_proxy'):
return url
ckan_url = config.get_value(u'ckan.site_url')
scheme = urlparse(url).scheme
compare_domains = datapreview.compare_domains
if not compare_domains([ckan_url, url]) and scheme in proxy_schemes:
url = h.url_for(u'resource_proxy.proxy_view',
id=data_dict[u'package'][u'name'],
resource_id=data_dict[u'resource'][u'id'])
log.info(u'Proxified url is {0}'.format(url))
return url
def test_user_objects_in_c_sysadmin(self):
'''
A sysadmin user request will have expected user objects added to
request.
'''
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
app = self._get_test_app()
user = factories.Sysadmin()
test_user_obj = model.User.by_name(user['name'])
resp = app.get(
'/from_pylons_extension_before_map',
extra_environ={'REMOTE_USER': user['name'].encode('ascii')})
# tmpl_context available on response
eq_(resp.tmpl_context.user, user['name'])
eq_(resp.tmpl_context.userobj, test_user_obj)
eq_(resp.tmpl_context.author, user['name'])
eq_(resp.tmpl_context.remote_addr, 'Unknown IP Address')
p.unload('test_routing_plugin')
def test_ask_around_pylons_extension_route_post_using_get(self):
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
app = self._get_test_app()
# We want our CKAN app, not the WebTest one
app = app.app
environ = {
'PATH_INFO': '/from_pylons_extension_before_map_post_only',
'REQUEST_METHOD': 'GET',
}
wsgiref.util.setup_testing_defaults(environ)
answers = app.ask_around(environ)
# We are going to get an answer from Pylons, but just because it will
# match the catch-all template route, hence the `core` origin.
eq_(answers, [(False, 'flask_app'), (True, 'pylons_app', 'core')])
p.unload('test_routing_plugin')
def test_api_info(self):
dataset = factories.Dataset()
resource = factories.Resource(
id='588dfa82-760c-45a2-b78a-e3bc314a4a9b',
package_id=dataset['id'],
datastore_active=True)
# the 'API info' is seen on the resource_read page, a snippet loaded by
# javascript via data_api_button.html
url = template_helpers.url_for(controller='api',
action='snippet',
ver=1,
snippet_path='api_info.html',
resource_id=resource['id'])
if not p.plugin_loaded('datastore'):
p.load('datastore')
app = self._get_test_app()
page = app.get(url, status=200)
p.unload('datastore')
# check we built all the urls ok
expected_urls = (
'http://test.ckan.net/api/3/action/datastore_create',
'http://test.ckan.net/api/3/action/datastore_upsert',
'<code>http://test.ckan.net/api/3/action/datastore_search',
'http://test.ckan.net/api/3/action/datastore_search_sql',
'http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&limit=5',
'http://test.ckan.net/api/3/action/datastore_search?q=jones&resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b',
'http://test.ckan.net/api/3/action/datastore_search_sql?sql=SELECT * from "588dfa82-760c-45a2-b78a-e3bc314a4a9b" WHERE title LIKE 'jones'',
"url: 'http://test.ckan.net/api/3/action/datastore_search'",
"http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&limit=5&q=title:jones",
)
for url in expected_urls:
assert url in page, url
def test_ask_around_pylons_extension_route_get_after_map(self):
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
app = self._get_test_app()
# We want our CKAN app, not the WebTest one
app = app.app
environ = {
'PATH_INFO': '/from_pylons_extension_after_map',
'REQUEST_METHOD': 'GET',
}
wsgiref.util.setup_testing_defaults(environ)
answers = app.ask_around('can_handle_request', environ)
eq_(len(answers), 1)
eq_(answers[0][0], True)
eq_(answers[0][1], 'pylons_app')
eq_(answers[0][2], 'extension')
p.unload('test_routing_plugin')
def before_index(self, pkg_dict):
if plugins.plugin_loaded('ga-report'):
pkg_dict['extras_ga_view_count'] = spc_utils.ga_view_count(
pkg_dict['name'])
topic_str = pkg_dict.get('thematic_area_string', '[]')
if isinstance(topic_str, string_types):
pkg_dict['topic'] = json.loads(topic_str)
else:
pkg_dict['topic'] = topic_str
pkg_dict.update(extras_five_star_rating=spc_utils.count_stars(
json.loads(pkg_dict['validated_data_dict']
) if 'validated_data_dict' in pkg_dict else pkg_dict))
pkg_dict.update(_five_star_rating=spc_utils.count_stars(
json.loads(pkg_dict['validated_data_dict']
) if 'validated_data_dict' in pkg_dict else pkg_dict))
if isinstance(pkg_dict.get('member_countries', '[]'), string_types):
pkg_dict['member_countries'] = spc_helpers.countries_list(
pkg_dict.get('member_countries', '[]'))
# Otherwise you'll get `immense field` error from SOLR
pkg_dict.pop('data_quality_info', None)
return pkg_dict
def datastore_create(context, data_dict):
'''Adds a new table to the DataStore.
The datastore_create action allows you to post JSON data to be
stored against a resource. This endpoint also supports altering tables,
aliases and indexes and bulk insertion. This endpoint can be called
multiple times to initially insert more data, add fields, change the
aliases or indexes as well as the primary keys.
To create an empty datastore resource and a CKAN resource at the same time,
provide ``resource`` with a valid ``package_id`` and omit the
``resource_id``.
If you want to create a datastore resource from the content of a file,
provide ``resource`` with a valid ``url``.
See :ref:`fields` and :ref:`records` for details on how to lay out records.
:param resource_id: resource id that the data is going to be stored
against.
:type resource_id: string
:param force: set to True to edit a read-only resource
:type force: bool (optional, default: False)
:param resource: resource dictionary that is passed to
:meth:`~ckan.logic.action.create.resource_create`.
Use instead of ``resource_id`` (optional)
:type resource: dictionary
:param aliases: names for read only aliases of the resource. (optional)
:type aliases: list or comma separated string
:param fields: fields/columns and their extra metadata. (optional)
:type fields: list of dictionaries
:param records: the data, eg: [{"dob": "2005", "some_stuff": ["a", "b"]}]
(optional)
:type records: list of dictionaries
:param primary_key: fields that represent a unique key (optional)
:type primary_key: list or comma separated string
:param indexes: indexes on table (optional)
:type indexes: list or comma separated string
:param triggers: trigger functions to apply to this table on update/insert.
functions may be created with
:meth:`~ckanext.datastore.logic.action.datastore_function_create`.
eg: [
{"function": "trigger_clean_reference"},
{"function": "trigger_check_codes"}]
:type triggers: list of dictionaries
:param calculate_record_count: updates the stored count of records, used to
optimize datastore_search in combination with the
`total_estimation_threshold` parameter. If doing a series of requests
to change a resource, you only need to set this to True on the last
request.
:type calculate_record_count: bool (optional, default: False)
Please note that setting the ``aliases``, ``indexes`` or ``primary_key``
replaces the existing aliases or constraints. Setting ``records`` appends
the provided records to the resource.
**Results:**
:returns: The newly created data object, excluding ``records`` passed.
:rtype: dictionary
See :ref:`fields` and :ref:`records` for details on how to lay out records.
'''
backend = DatastoreBackend.get_active_backend()
schema = context.get('schema', dsschema.datastore_create_schema())
records = data_dict.pop('records', None)
resource = data_dict.pop('resource', None)
data_dict, errors = _validate(data_dict, schema, context)
resource_dict = None
if records:
data_dict['records'] = records
if resource:
data_dict['resource'] = resource
if errors:
raise p.toolkit.ValidationError(errors)
p.toolkit.check_access('datastore_create', context, data_dict)
if 'resource' in data_dict and 'resource_id' in data_dict:
raise p.toolkit.ValidationError(
{'resource': ['resource cannot be used with resource_id']})
if 'resource' not in data_dict and 'resource_id' not in data_dict:
raise p.toolkit.ValidationError(
{'resource_id': ['resource_id or resource required']})
if 'resource' in data_dict:
has_url = 'url' in data_dict['resource']
# A datastore only resource does not have a url in the db
data_dict['resource'].setdefault('url', '_datastore_only_resource')
resource_dict = p.toolkit.get_action('resource_create')(
context, data_dict['resource'])
data_dict['resource_id'] = resource_dict['id']
# create resource from file
if has_url:
if not p.plugin_loaded('datapusher'):
raise p.toolkit.ValidationError(
{'resource': ['The datapusher has to be enabled.']})
p.toolkit.get_action('datapusher_submit')(
context, {
'resource_id': resource_dict['id'],
'set_url_type': True
})
# since we'll overwrite the datastore resource anyway, we
# don't need to create it here
return
# create empty resource
else:
# no need to set the full url because it will be set in before_show
resource_dict['url_type'] = 'datastore'
p.toolkit.get_action('resource_update')(context, resource_dict)
else:
if not data_dict.pop('force', False):
resource_id = data_dict['resource_id']
_check_read_only(context, resource_id)
# validate aliases
aliases = datastore_helpers.get_list(data_dict.get('aliases', []))
for alias in aliases:
if not datastore_helpers.is_valid_table_name(alias):
raise p.toolkit.ValidationError(
{'alias': [u'"{0}" is not a valid alias name'.format(alias)]})
try:
result = backend.create(context, data_dict)
except InvalidDataError as err:
raise p.toolkit.ValidationError(text_type(err))
if data_dict.get('calculate_record_count', False):
backend.calculate_record_count(data_dict['resource_id'])
# Set the datastore_active flag on the resource if necessary
model = _get_or_bust(context, 'model')
resobj = model.Resource.get(data_dict['resource_id'])
if resobj.extras.get('datastore_active') is not True:
log.debug('Setting datastore_active=True on resource {0}'.format(
resobj.id))
set_datastore_active_flag(model, data_dict, True)
result.pop('id', None)
result.pop('connection_url', None)
result.pop('records', None)
return result
def teardown_class(cls):
super(BaseTestInterfaces, cls).teardown_class()
if p.plugin_loaded('test_validation_plugin'):
p.unload('test_validation_plugin')
def datastore_create(context, data_dict):
'''Adds a new table to the DataStore.
The datastore_create action allows you to post JSON data to be
stored against a resource. This endpoint also supports altering tables,
aliases and indexes and bulk insertion. This endpoint can be called multiple
times to initially insert more data, add fields, change the aliases or indexes
as well as the primary keys.
To create an empty datastore resource and a CKAN resource at the same time,
provide ``resource`` with a valid ``package_id`` and omit the ``resource_id``.
If you want to create a datastore resource from the content of a file,
provide ``resource`` with a valid ``url``.
See :ref:`fields` and :ref:`records` for details on how to lay out records.
:param resource_id: resource id that the data is going to be stored against.
:type resource_id: string
:param force: set to True to edit a read-only resource
:type force: bool (optional, default: False)
:param resource: resource dictionary that is passed to
:meth:`~ckan.logic.action.create.resource_create`.
Use instead of ``resource_id`` (optional)
:type resource: dictionary
:param aliases: names for read only aliases of the resource. (optional)
:type aliases: list or comma separated string
:param fields: fields/columns and their extra metadata. (optional)
:type fields: list of dictionaries
:param records: the data, eg: [{"dob": "2005", "some_stuff": ["a", "b"]}] (optional)
:type records: list of dictionaries
:param primary_key: fields that represent a unique key (optional)
:type primary_key: list or comma separated string
:param indexes: indexes on table (optional)
:type indexes: list or comma separated string
Please note that setting the ``aliases``, ``indexes`` or ``primary_key`` replaces the exising
aliases or constraints. Setting ``records`` appends the provided records to the resource.
**Results:**
:returns: The newly created data object.
:rtype: dictionary
See :ref:`fields` and :ref:`records` for details on how to lay out records.
'''
schema = context.get('schema', dsschema.datastore_create_schema())
records = data_dict.pop('records', None)
resource = data_dict.pop('resource', None)
data_dict, errors = _validate(data_dict, schema, context)
resource_dict = None
if records:
data_dict['records'] = records
if resource:
data_dict['resource'] = resource
if errors:
raise p.toolkit.ValidationError(errors)
p.toolkit.check_access('datastore_ts_create', context, data_dict)
if 'resource' in data_dict and 'resource_id' in data_dict:
raise p.toolkit.ValidationError({
'resource': ['resource cannot be used with resource_id']
})
if not 'resource' in data_dict and not 'resource_id' in data_dict:
raise p.toolkit.ValidationError({
'resource_id': ['resource_id or resource required']
})
if 'resource' in data_dict:
has_url = 'url' in data_dict['resource']
if 'retention' in data_dict['resource']:
try:
retention = int(data_dict['resource']['retention'])
if retention < 1 or retention > 100:
raise Exception()
except:
raise p.toolkit.ValidationError({'resource': [
'Retention must be an integer from 1-100']})
# A datastore only resource does not have a url in the db
data_dict['resource'].setdefault('url', '_datastore_only_resource')
resource_dict = p.toolkit.get_action('resource_create')(
context, data_dict['resource'])
data_dict['resource_id'] = resource_dict['id']
# create resource from file
if has_url:
if not p.plugin_loaded('datapusher'):
raise p.toolkit.ValidationError({'resource': [
'The datapusher has to be enabled.']})
p.toolkit.get_action('datapusher_submit')(context, {
'resource_id': resource_dict['id'],
'set_url_type': True
})
# since we'll overwrite the datastore resource anyway, we
# don't need to create it here
return
# create empty resource
else:
# no need to set the full url because it will be set in before_show
resource_dict['url_type'] = 'datastore'
p.toolkit.get_action('resource_update')(context, resource_dict)
else:
if not data_dict.pop('force', False):
resource_id = data_dict['resource_id']
_check_read_only(context, resource_id)
data_dict['connection_url'] = config['ckan.datastore.write_url']
# validate aliases
aliases = datastore_helpers.get_list(data_dict.get('aliases', []))
for alias in aliases:
if not db._is_valid_table_name(alias):
raise p.toolkit.ValidationError({
'alias': [u'"{0}" is not a valid alias name'.format(alias)]
})
# create a private datastore resource, if necessary
model = _get_or_bust(context, 'model')
resource = model.Resource.get(data_dict['resource_id'])
legacy_mode = 'ckan.datastore.read_url' not in config
if not legacy_mode and resource.package.private:
data_dict['private'] = True
try:
result = db.create(context, data_dict)
except db.InvalidDataError as err:
raise p.toolkit.ValidationError(unicode(err))
# Set the datastore_active flag on the resource if necessary
if resource.extras.get('datastore_active') is not True:
log.debug(
'Setting datastore_active=True on resource {0}'.format(resource.id)
)
# issue #3245: race condition
update_dict = {'datastore_active': True}
# get extras(for entity update) and package_id(for search index update)
res_query = model.Session.query(
model.resource_table.c.extras,
model.resource_table.c.package_id
).filter(
model.Resource.id == data_dict['resource_id']
)
extras, package_id = res_query.one()
# update extras in database for record and its revision
extras.update(update_dict)
res_query.update({'extras': extras}, synchronize_session=False)
model.Session.query(model.resource_revision_table).filter(
model.ResourceRevision.id == data_dict['resource_id'],
model.ResourceRevision.current is True
).update({'extras': extras}, synchronize_session=False)
model.Session.commit()
# get package with updated resource from solr
# find changed resource, patch it and reindex package
psi = search.PackageSearchIndex()
solr_query = search.PackageSearchQuery()
q = {
'q': 'id:"{0}"'.format(package_id),
'fl': 'data_dict',
'wt': 'json',
'fq': 'site_id:"%s"' % config.get('ckan.site_id'),
'rows': 1
}
for record in solr_query.run(q)['results']:
solr_data_dict = json.loads(record['data_dict'])
for resource in solr_data_dict['resources']:
if resource['id'] == data_dict['resource_id']:
resource.update(update_dict)
psi.index_package(solr_data_dict)
break
result.pop('id', None)
result.pop('private', None)
result.pop('connection_url')
datastore_helpers.remove_autogen(result)
return result
def test_with_plugins_is_able_to_run_with_stats():
assert plugins.plugin_loaded(u"stats")
def _inject_views_into_page(self, _page):
# this is a good proxy to a version of CKAN with views enabled.
if not p.plugin_loaded('image_view'):
return
try:
import lxml
import lxml.html
except ImportError:
return
try:
root = lxml.html.fromstring(_page['content'])
# Return if any errors are found while parsing the content
except (lxml.etree.XMLSyntaxError,
lxml.etree.ParserError):
return
for element in root.findall('.//iframe'):
embed_element = element.attrib.pop('data-ckan-view-embed', None)
if not embed_element:
continue
element.tag = 'div'
error = None
try:
iframe_src = element.attrib.pop('src', '')
width = element.attrib.pop('width', '80')
if not width.endswith('%') and not width.endswith('px'):
width = width + 'px'
height = element.attrib.pop('height', '80')
if not height.endswith('%') and not height.endswith('px'):
height = height + 'px'
align = element.attrib.pop('align', 'none')
style = "width: %s; height: %s; float: %s; overflow: auto; vertical-align:middle; position:relative" % (width, height, align)
element.attrib['style'] = style
element.attrib['class'] = 'pages-embed'
view = p.toolkit.get_action('resource_view_show')({}, {'id': iframe_src[-36:]})
context = {}
resource = p.toolkit.get_action('resource_show')(context, {'id': view['resource_id']})
package_id = context['resource'].resource_group.package_id
package = p.toolkit.get_action('package_show')(context, {'id': package_id})
except p.toolkit.ObjectNotFound:
error = _('ERROR: View not found {view_id}'.format(view_id=iframe_src ))
if error:
resource_view_html = '<h4> %s </h4>' % error
elif not helpers.resource_view_is_iframed(view):
resource_view_html = helpers.rendered_resource_view(view, resource, package)
else:
src = helpers.url_for(qualified=True, controller='package', action='resource_view', id=package['name'], resource_id=resource['id'], view_id=view['id'])
message = _('Your browser does not support iframes.')
resource_view_html = '<iframe src="{src}" frameborder="0" width="100%" height="100%" style="display:block"> <p>{message}</p> </iframe>'.format(src=src, message=message)
view_element = lxml.html.fromstring(resource_view_html)
element.append(view_element)
new_content = lxml.html.tostring(root)
if new_content.startswith('<div>') and new_content.endswith('</div>'):
# lxml will add a <div> tag to text that starts with an HTML tag,
# which will cause the rendering to fail
new_content = new_content[5:-6]
elif new_content.startswith('<p>') and new_content.endswith('</p>'):
# lxml will add a <p> tag to plain text snippet, which will cause the
# rendering to fail
new_content = new_content[3:-4]
_page['content'] = new_content
def update_config(self, config):
p.toolkit.add_public_directory(config, 'public')
p.toolkit.add_template_directory(config, 'templates')
p.toolkit.add_resource('public', 'ckanext-spatial')
config['ckan.resource_proxy_enabled'] = p.plugin_loaded('resource_proxy')
def teardown_class(cls):
if plugins.plugin_loaded('example_igroupform_organization'):
plugins.unload('example_igroupform_organization')
super(TestOrganizationController, cls).teardown_class()
def setup(self):
if p.plugin_loaded('datastore_ts'):
p.unload('timeseries')
if p.plugin_loaded('sample_datastore_plugin'):
p.unload('sample_datastore_plugin')
def teardown_class(cls):
if plugins.plugin_loaded('example_igroupform_default_group_type'):
plugins.unload('example_igroupform_default_group_type')
super(TestGroupControllerEdit_DefaultGroupType, cls).teardown_class()
def setup(self):
if p.plugin_loaded('datastore'):
p.unload('datastore')
if p.plugin_loaded('sample_datastore_plugin'):
p.unload('sample_datastore_plugin')
def setup_class(cls):
if not p.plugin_loaded('image_view'):
p.load('image_view')
def get(self,
package_type,
id,
resource_id,
view_id=None,
post_extra=None):
context, extra_vars = self._prepare(id, resource_id)
to_preview = extra_vars[u'to_preview']
if post_extra:
extra_vars.update(post_extra)
package_type = _get_package_type(id)
data = extra_vars[u'data'] if u'data' in extra_vars else None
if data and u'view_type' in data:
view_type = data.get(u'view_type')
else:
view_type = request.args.get(u'view_type')
# view_id exists only when updating
if view_id:
if not data or not view_type:
try:
view_data = get_action(u'resource_view_show')(
context, {
u'id': view_id
})
view_type = view_data[u'view_type']
if data:
data.update(view_data)
else:
data = view_data
except (NotFound, NotAuthorized):
return base.abort(404, _(u'View not found'))
# might as well preview when loading good existing view
if not extra_vars[u'errors']:
to_preview = True
data[u'view_type'] = view_type
view_plugin = lib_datapreview.get_view_plugin(view_type)
if not view_plugin:
return base.abort(404, _(u'View Type Not found'))
_setup_template_variables(context, {u'id': id},
package_type=package_type)
data_dict = {
u'package': extra_vars[u'pkg_dict'],
u'resource': extra_vars[u'resource'],
u'resource_view': data
}
view_template = view_plugin.view_template(context, data_dict)
form_template = view_plugin.form_template(context, data_dict)
extra_vars.update({
u'form_template':
form_template,
u'view_template':
view_template,
u'data':
data,
u'to_preview':
to_preview,
u'datastore_available':
plugins.plugin_loaded(u'datastore')
})
extra_vars.update(
view_plugin.setup_template_variables(context, data_dict) or {})
extra_vars.update(data_dict)
if view_id:
return base.render(u'package/edit_view.html', extra_vars)
return base.render(u'package/new_view.html', extra_vars)
def teardown_class(cls):
if plugins.plugin_loaded('example_igroupform'):
plugins.unload('example_igroupform')
super(TestGroupController, cls).teardown_class()
def setup_class(cls):
if not p.plugin_loaded('datastore'):
p.load('datastore')
def setup_class(cls):
super(TestExampleCustomEmailsPlugin, cls).setup_class()
if not plugins.plugin_loaded('example_theme_custom_emails'):
plugins.load('example_theme_custom_emails')
def load_environment(conf):
"""
Configure the Pylons environment via the ``pylons.config`` object. This
code should only need to be run once.
"""
if six.PY2:
# this must be run at a time when the env is semi-setup, thus inlined
# here. Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = getattr(PylonsApp.find_controller,
'_old_find_controller',
PylonsApp.find_controller)
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
if hasattr(ep, 'resolve'):
# setuptools >= 10.2
mycontroller = ep.resolve()
else:
# setuptools >= 11.3
mycontroller = ep.load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
find_controller._old_find_controller = find_controller_generic
PylonsApp.find_controller = find_controller
os.environ['CKAN_CONFIG'] = conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
valid_base_public_folder_names = ['public']
static_files = conf.get('ckan.base_public_folder', 'public')
conf['ckan.base_public_folder'] = static_files
if static_files not in valid_base_public_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_public_folder. '
'Possible values are: "public".')
log.info('Loading static files from %s' % static_files)
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, static_files),
templates=[])
# Initialize main CKAN config object
config.update(conf)
if six.PY2:
# Initialize Pylons own config object
pylons_config.init_app(conf['global_conf'],
conf,
package='ckan',
paths=paths)
# Update the main CKAN config object with the Pylons specific stuff,
# as it is quite hard to keep them separated. This should be removed
# once Pylons support is dropped
config.update(pylons_config)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = [
'^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
# load all CKAN plugins
p.load_all()
if not p.plugin_loaded('managed_search_schema'):
search.check_solr_schema_version()
# Check Redis availability
if not is_redis_available():
log.critical('Could not connect to Redis.')
app_globals.reset()
# issue #3260: remove idle transaction
# Session that was used for getting all config params nor committed,
# neither removed and we have idle connection as result
model.Session.commit()
# Build JavaScript translations. Must be done after plugins have
# been loaded.
build_js_translations()
def setup_class(cls):
if not p.plugin_loaded('image_view'):
p.load('image_view')
helpers.reset_db()
def setup_class(cls):
super(BaseTestInterfaces, cls).setup_class()
if not p.plugin_loaded('test_validation_plugin'):
p.load('test_validation_plugin')
def teardown(self):
if p.plugin_loaded("sample_datastore_plugin"):
p.unload("sample_datastore_plugin")
if p.plugin_loaded("datastore"):
p.unload("datastore")
def teardown_class(cls):
if p.plugin_loaded('recline_view'):
p.unload('recline_view')
if p.plugin_loaded('datastore'):
p.unload('datastore')
def setup_class(cls):
super(TestWebPageView, cls).setup_class()
if not p.plugin_loaded('webpage_view'):
p.load('webpage_view')
def datastore_create(context, data_dict):
'''Adds a new table to the DataStore.
The datastore_create action allows you to post JSON data to be
stored against a resource. This endpoint also supports altering tables,
aliases and indexes and bulk insertion. This endpoint can be called multiple
times to initially insert more data, add fields, change the aliases or indexes
as well as the primary keys.
To create an empty datastore resource and a CKAN resource at the same time,
provide ``resource`` with a valid ``package_id`` and omit the ``resource_id``.
If you want to create a datastore resource from the content of a file,
provide ``resource`` with a valid ``url``.
See :ref:`fields` and :ref:`records` for details on how to lay out records.
:param resource_id: resource id that the data is going to be stored against.
:type resource_id: string
:param force: set to True to edit a read-only resource
:type force: bool (optional, default: False)
:param resource: resource dictionary that is passed to
:meth:`~ckan.logic.action.create.resource_create`.
Use instead of ``resource_id`` (optional)
:type resource: dictionary
:param aliases: names for read only aliases of the resource. (optional)
:type aliases: list or comma separated string
:param fields: fields/columns and their extra metadata. (optional)
:type fields: list of dictionaries
:param records: the data, eg: [{"dob": "2005", "some_stuff": ["a", "b"]}] (optional)
:type records: list of dictionaries
:param primary_key: fields that represent a unique key (optional)
:type primary_key: list or comma separated string
:param indexes: indexes on table (optional)
:type indexes: list or comma separated string
Please note that setting the ``aliases``, ``indexes`` or ``primary_key`` replaces the exising
aliases or constraints. Setting ``records`` appends the provided records to the resource.
**Results:**
:returns: The newly created data object.
:rtype: dictionary
See :ref:`fields` and :ref:`records` for details on how to lay out records.
'''
schema = context.get('schema', dsschema.datastore_create_schema())
records = data_dict.pop('records', None)
resource = data_dict.pop('resource', None)
data_dict, errors = _validate(data_dict, schema, context)
if records:
data_dict['records'] = records
if resource:
data_dict['resource'] = resource
if errors:
raise p.toolkit.ValidationError(errors)
p.toolkit.check_access('datastore_create', context, data_dict)
if 'resource' in data_dict and 'resource_id' in data_dict:
raise p.toolkit.ValidationError(
{'resource': ['resource cannot be used with resource_id']})
if not 'resource' in data_dict and not 'resource_id' in data_dict:
raise p.toolkit.ValidationError(
{'resource_id': ['resource_id or resource required']})
if 'resource' in data_dict:
has_url = 'url' in data_dict['resource']
# A datastore only resource does not have a url in the db
data_dict['resource'].setdefault('url', '_datastore_only_resource')
res = p.toolkit.get_action('resource_create')(context,
data_dict['resource'])
data_dict['resource_id'] = res['id']
# create resource from file
if has_url:
if not p.plugin_loaded('datapusher'):
raise p.toolkit.ValidationError(
{'resource': ['The datapusher has to be enabled.']})
p.toolkit.get_action('datapusher_submit')(context, {
'resource_id': res['id'],
'set_url_type': True
})
# since we'll overwrite the datastore resource anyway, we
# don't need to create it here
return
# create empty resource
else:
# no need to set the full url because it will be set in before_show
res['url_type'] = 'datastore'
p.toolkit.get_action('resource_update')(context, res)
else:
if not data_dict.pop('force', False):
resource_id = data_dict['resource_id']
_check_read_only(context, resource_id)
data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
# validate aliases
aliases = datastore_helpers.get_list(data_dict.get('aliases', []))
for alias in aliases:
if not db._is_valid_table_name(alias):
raise p.toolkit.ValidationError(
{'alias': [u'"{0}" is not a valid alias name'.format(alias)]})
# create a private datastore resource, if necessary
model = _get_or_bust(context, 'model')
resource = model.Resource.get(data_dict['resource_id'])
legacy_mode = 'ckan.datastore.read_url' not in pylons.config
if not legacy_mode and resource.package.private:
data_dict['private'] = True
try:
result = db.create(context, data_dict)
except db.InvalidDataError as err:
raise p.toolkit.ValidationError(str(err))
result.pop('id', None)
result.pop('private', None)
result.pop('connection_url')
return result
def setup(self):
if p.plugin_loaded("datastore"):
p.unload("datastore")
if p.plugin_loaded("sample_datastore_plugin"):
p.unload("sample_datastore_plugin")
