def test_translation_works_on_flask_and_pylons(self):
app = helpers._get_test_app()
if not p.plugin_loaded(u'test_routing_plugin'):
p.load(u'test_routing_plugin')
try:
plugin = p.get_plugin(u'test_routing_plugin')
app.flask_app.register_extension_blueprint(
plugin.get_blueprint())
resp = app.get(u'/flask_translated')
eq_(resp.body, u'Dataset')
resp = app.get(u'/es/flask_translated')
eq_(resp.body, u'Conjunto de datos')
resp = app.get(u'/pylons_translated')
eq_(resp.body, u'Groups')
resp = app.get(u'/es/pylons_translated')
eq_(resp.body, u'Grupos')
finally:
if p.plugin_loaded(u'test_routing_plugin'):
p.unload(u'test_routing_plugin')
def test_read_plugin_hook(self):
plugins.load("test_group_plugin")
name = u"david"
offset = url_for(controller="group", action="read", id=name)
res = self.app.get(offset, status=200, extra_environ={"REMOTE_USER": "******"})
p = plugins.get_plugin("test_group_plugin")
assert p.calls["read"] == 1, p.calls
plugins.unload("test_group_plugin")
def setup(self):
self.app = helpers._get_test_app()
# Install plugin and register its blueprint
if not plugins.plugin_loaded(u'example_flask_iblueprint'):
plugins.load(u'example_flask_iblueprint')
plugin = plugins.get_plugin(u'example_flask_iblueprint')
self.app.flask_app.register_extension_blueprint(plugin.get_blueprint())
def test_search_plugin_hooks(self):
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
offset = url_for(controller='package', action='search')
res = self.app.get(offset)
# get redirected ...
assert plugin.calls['before_search'] == 1, plugin.calls
assert plugin.calls['after_search'] == 1, plugin.calls
plugins.unload('test_package_controller_plugin')
def test_read_plugin_hook(self):
plugins.load('test_group_plugin')
name = u'david'
offset = url_for(controller='group', action='read', id=name)
res = self.app.get(offset, status=200,
extra_environ={'REMOTE_USER': '******'})
p = plugins.get_plugin('test_group_plugin')
assert p.calls['read'] == 1, p.calls
plugins.unload('test_group_plugin')
def setup(self):
self.app = helpers._get_test_app()
# Install plugin and register its blueprint
if not p.plugin_loaded(u'test_flash_plugin'):
p.load(u'test_flash_plugin')
plugin = p.get_plugin(u'test_flash_plugin')
self.app.flask_app.register_extension_blueprint(
plugin.get_blueprint())
def setup_class(cls):
super(TestFlaskUserIdentifiedInRequest, cls).setup_class()
cls.app = cls._get_test_app()
cls.flask_app = cls.app.flask_app
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
plugin = p.get_plugin('test_routing_plugin')
cls.flask_app.register_extension_blueprint(
plugin.get_blueprint())
def test_harvest_before_download_extension_point_gets_called(self):
plugin = p.get_plugin('test_rdf_harvester')
harvest_source = self._create_harvest_source()
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['before_download'], 1)
def test_read_plugin_hook(self):
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
name = u'annakarenina'
offset = url_for(controller='package', action='read', id=name)
res = self.app.get(offset)
assert plugin.calls['read'] == 1, plugin.calls
assert plugin.calls['after_show'] == 1, plugin.calls
plugins.unload('test_package_controller_plugin')
def setup_class(cls):
super(TestCORSFlask, cls).setup_class()
cls.app = cls._get_test_app()
flask_app = cls.app.flask_app
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
plugin = p.get_plugin('test_routing_plugin')
flask_app.register_blueprint(plugin.get_blueprint(),
prioritise_rules=True)
def test_new_plugin_hook(self):
plugins.load("test_group_plugin")
offset = url_for(controller="group", action="new")
res = self.app.get(offset, status=200, extra_environ={"REMOTE_USER": "******"})
form = res.forms["group-edit"]
form["name"] = "hahaha"
form["title"] = "huhuhu"
res = form.submit("save", status=302, extra_environ={"REMOTE_USER": "******"})
p = plugins.get_plugin("test_group_plugin")
assert p.calls["create"] == 1, p.calls
plugins.unload("test_group_plugin")
def test_edit_plugin_hook(self):
plugins.load("test_group_plugin")
offset = url_for(controller="group", action="edit", id=self.groupname)
res = self.app.get(offset, status=200, extra_environ={"REMOTE_USER": "******"})
form = res.forms["group-edit"]
group = model.Group.by_name(self.groupname)
form["title"] = "huhuhu"
res = form.submit("save", status=302, extra_environ={"REMOTE_USER": "******"})
p = plugins.get_plugin("test_group_plugin")
assert p.calls["edit"] == 1, p.calls
plugins.unload("test_group_plugin")
def test_plugins_load(self):
config_plugins = config['ckan.plugins']
config['ckan.plugins'] = 'mapper_plugin routes_plugin'
plugins.load_all(config)
# synchronous_search automatically gets loaded
current_plugins = set([plugins.get_plugin(p) for p in ['mapper_plugin', 'routes_plugin', 'synchronous_search'] + find_system_plugins()])
assert PluginGlobals.env().services == current_plugins
# cleanup
config['ckan.plugins'] = config_plugins
plugins.load_all(config)
def test_edit_plugin_hook(self):
plugins.load('test_group_plugin')
offset = url_for(controller='group', action='edit', id=self.groupname)
res = self.app.get(offset, status=200,
extra_environ={'REMOTE_USER': '******'})
form = res.forms['group-edit']
group = model.Group.by_name(self.groupname)
form['title'] = "huhuhu"
res = form.submit('save', status=302,
extra_environ={'REMOTE_USER': '******'})
p = plugins.get_plugin('test_group_plugin')
assert p.calls['edit'] == 1, p.calls
plugins.unload('test_group_plugin')
def test_new_plugin_hook(self):
plugins.load('test_group_plugin')
offset = url_for(controller='group', action='new')
res = self.app.get(offset, status=200,
extra_environ={'REMOTE_USER': '******'})
form = res.forms['group-edit']
form['name'] = "hahaha"
form['title'] = "huhuhu"
res = form.submit('save', status=302,
extra_environ={'REMOTE_USER': '******'})
p = plugins.get_plugin('test_group_plugin')
assert p.calls['create'] == 1, p.calls
plugins.unload('test_group_plugin')
def test_url_for_flask_request_using_pylons_url_for(self):
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
plugin = p.get_plugin('test_routing_plugin')
self.app.flask_app.register_extension_blueprint(
plugin.get_blueprint())
res = self.app.get('/flask_route_pylons_url_for')
assert u'This URL was generated by Pylons' in res.ubody
assert u'/from_pylons_extension_before_map' in res.ubody
p.unload('test_routing_plugin')
def test_new_plugin_hook(self):
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
offset = url_for('dataset.new')
res = self.app.get(offset, extra_environ=self.extra_environ_tester)
new_name = u'plugged'
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = new_name
res = fv.submit('save', extra_environ=self.extra_environ_tester)
# get redirected ...
assert plugin.calls['edit'] == 0, plugin.calls
assert plugin.calls['create'] == 1, plugin.calls
plugins.unload('test_package_controller_plugin')
def test_new_plugin_hook(self):
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
offset = url_for(controller='package', action='new')
res = self.app.get(offset, extra_environ=self.extra_environ_tester)
new_name = u'plugged'
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = new_name
res = fv.submit('save', extra_environ=self.extra_environ_tester)
# get redirected ...
assert plugin.calls['edit'] == 0, plugin.calls
assert plugin.calls['create'] == 1, plugin.calls
plugins.unload('test_package_controller_plugin')
def setup_class(cls):
plugins.load('test_resource_preview', 'test_json_resource_preview')
cls.plugin = plugins.get_plugin('test_resource_preview')
create_test_data.CreateTestData.create()
cls.package = model.Package.get('annakarenina')
cls.resource = cls.package.resources[0]
cls.url = h.url_for('resource.read',
id=cls.package.name,
resource_id=cls.resource.id)
cls.preview_url = h.url_for('resource.datapreview',
id=cls.package.id,
resource_id=cls.resource.id)
def test_url_for_flask_request_using_pylons_url_for(self):
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
plugin = p.get_plugin('test_routing_plugin')
self.app.flask_app.register_extension_blueprint(
plugin.get_blueprint())
res = self.app.get('/flask_route_pylons_url_for')
assert u'This URL was generated by Pylons' in res.ubody
assert u'/from_pylons_extension_before_map' in res.ubody
p.unload('test_routing_plugin')
def test_harvest_import_extensions_point_gets_called(self):
plugin = p.get_plugin('test_rdf_harvester')
url = self.rdf_mock_url
content = self.rdf_content
content_type = self.rdf_content_type
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET, url,
body=content, content_type=content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD, url,
status=405, content_type=content_type)
harvest_source = self._create_harvest_source(url)
# First run, will create two datasets as previously tested
self._run_full_job(harvest_source['id'], num_objects=2)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Get the harvest source with the udpated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_(last_job_status['status'], 'Finished')
eq_(plugin.calls['before_create'], 2)
eq_(plugin.calls['after_create'], 2)
eq_(plugin.calls['before_update'], 0)
eq_(plugin.calls['after_update'], 0)
# Mock an update in the remote file
new_file = content.replace('Example dataset 1',
'Example dataset 1 (updated)')
httpretty.register_uri(httpretty.GET, url,
body=new_file, content_type=content_type)
# Run a second job
self._run_full_job(harvest_source['id'], num_objects=2)
eq_(plugin.calls['before_create'], 2)
eq_(plugin.calls['after_create'], 2)
eq_(plugin.calls['before_update'], 2)
eq_(plugin.calls['after_update'], 2)
def test_flask_extension_route_is_served_by_flask(self):
app = self._get_test_app()
# Install plugin and register its blueprint
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
plugin = p.get_plugin('test_routing_plugin')
app.flask_app.register_extension_blueprint(plugin.get_blueprint())
res = app.get('/simple_flask')
eq_(res.environ['ckan.app'], 'flask_app')
p.unload('test_routing_plugin')
def test_after_create_plugin_hook(self, env_user, app):
plugin = plugins.get_plugin("test_package_controller_plugin")
offset = url_for("dataset.new")
new_name = u"plugged2"
res = app.post(offset,
extra_environ=env_user,
data={
"name": new_name,
"save": ""
},
follow_redirects=False)
assert plugin.calls["after_update"] == 0, plugin.calls
assert plugin.calls["after_create"] == 1, plugin.calls
assert plugin.id_in_dict
def test_ipipe_notified(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
res_id = self._test_resource(url)['id']
update_resource(self.config, res_id, 'queue1')
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == res_id
def test_ipipe_notified(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
res_id = self._test_resource(url)['id']
update_resource(self.config, res_id, 'queue1')
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == res_id
def test_new_plugin_hook(self):
plugins.load('test_group_plugin')
offset = url_for(controller='group', action='new')
res = self.app.get(offset,
status=200,
extra_environ={'REMOTE_USER': '******'})
form = res.forms['group-edit']
form['name'] = "hahaha"
form['title'] = "huhuhu"
res = form.submit('save',
status=302,
extra_environ={'REMOTE_USER': '******'})
p = plugins.get_plugin('test_group_plugin')
assert p.calls['create'] == 1, p.calls
plugins.unload('test_group_plugin')
def test_flask_extension_route_is_served_by_flask(self):
app = self._get_test_app()
# Install plugin and register its blueprint
if not p.plugin_loaded('test_routing_plugin'):
p.load('test_routing_plugin')
plugin = p.get_plugin('test_routing_plugin')
app.flask_app.register_extension_blueprint(plugin.get_blueprint())
res = app.get('/simple_flask')
eq_(res.environ['ckan.app'], 'flask_app')
p.unload('test_routing_plugin')
def test_edit_plugin_hook(self):
plugins.load('test_group_plugin')
offset = url_for(controller='group', action='edit', id=self.groupname)
res = self.app.get(offset,
status=200,
extra_environ={'REMOTE_USER': '******'})
form = res.forms['group-edit']
group = model.Group.by_name(self.groupname)
form['title'] = "huhuhu"
res = form.submit('save',
status=302,
extra_environ={'REMOTE_USER': '******'})
p = plugins.get_plugin('test_group_plugin')
assert p.calls['edit'] == 1, p.calls
plugins.unload('test_group_plugin')
def test_plugins_load(self):
config_plugins = config['ckan.plugins']
config['ckan.plugins'] = 'mapper_plugin routes_plugin'
plugins.load_all(config)
# synchronous_search automatically gets loaded
current_plugins = set([
plugins.get_plugin(p)
for p in ['mapper_plugin', 'routes_plugin', 'synchronous_search'] +
find_system_plugins()
])
assert PluginGlobals.env().services == current_plugins
# cleanup
config['ckan.plugins'] = config_plugins
plugins.load_all(config)
def test_plugins_load(self):
config_plugins = config['ckan.plugins']
config['ckan.plugins'] = 'mapper_plugin routes_plugin'
plugins.load_all()
# synchronous_search automatically gets loaded
current_plugins = set([
plugins.get_plugin(p)
for p in ['mapper_plugin', 'routes_plugin', 'synchronous_search'] +
find_system_plugins()
])
assert set(plugins.core._PLUGINS_SERVICE.values()) == current_plugins
# cleanup
config['ckan.plugins'] = config_plugins
plugins.load_all()
def test_ipipe_notified(self, client):
url = client + '/?status=200&content=test&content-type=csv'
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
res_id = self._test_resource(url)['id']
update_resource(res_id, 'queue1')
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == res_id
def setup_class(cls):
plugins.load('test_resource_preview', 'test_json_resource_preview')
cls.plugin = plugins.get_plugin('test_resource_preview')
create_test_data.CreateTestData.create()
cls.package = model.Package.get('annakarenina')
cls.resource = cls.package.resources[0]
cls.url = h.url_for(controller='package',
action='resource_read',
id=cls.package.name,
resource_id=cls.resource.id)
cls.preview_url = h.url_for(controller='package',
action='resource_datapreview',
id=cls.package.id,
resource_id=cls.resource.id)
def test_delete(self):
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
offset = url_for('dataset.delete', id='warandpeace')
# Since organizations, any owned dataset can be edited/deleted by any
# user
self.app.post(offset, extra_environ=self.extra_environ_tester)
self.app.post(offset, extra_environ=self.extra_environ_admin)
assert model.Package.get('warandpeace').state == u'deleted'
assert plugin.calls['delete'] == 2
assert plugin.calls['after_delete'] == 2
plugins.unload('test_package_controller_plugin')
def test_delete(self):
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
offset = url_for('dataset.delete', id='warandpeace')
# Since organizations, any owned dataset can be edited/deleted by any
# user
self.app.post(offset, extra_environ=self.extra_environ_tester)
self.app.post(offset, extra_environ=self.extra_environ_admin)
assert model.Package.get('warandpeace').state == u'deleted'
assert plugin.calls['delete'] == 2
assert plugin.calls['after_delete'] == 2
plugins.unload('test_package_controller_plugin')
def test_harvest_with_before_create_raising_exception(self):
plugin = p.get_plugin('test_rdf_exception_harvester')
url = self.rdf_mock_url
content = self.rdf_content
content_type = self.rdf_content_type
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET, url,
body=content, content_type=content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD, url,
status=405, content_type=content_type)
harvest_source = self._create_harvest_source(url)
self._run_full_job(harvest_source['id'], num_objects=2)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Get the harvest source with the updated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_(last_job_status['status'], 'Finished')
assert ('Error importing dataset'
in last_job_status['object_error_summary'][0][0])
nose.tools.assert_dict_equal(
last_job_status['stats'],
{
'deleted': 0,
'added': 1,
'updated': 0,
'not modified': 0,
'errored': 1
}
)
eq_(plugin.calls['before_create'], 2)
eq_(plugin.calls['after_create'], 1)
eq_(plugin.calls['before_update'], 0)
eq_(plugin.calls['after_update'], 0)
def test_ipipe_notified(self, url):
testipipe = plugins.get_plugin("testipipe")
testipipe.reset()
res_id = self._test_resource(url)["id"]
# celery.send_task doesn't respect CELERY_ALWAYS_EAGER
res = update_resource.apply_async(args=[self.config, res_id, "queue1"])
res.get()
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == "archived"
assert queue == "queue1"
assert params.get("package_id") == None
assert params.get("resource_id") == res_id
def test_harvest_after_download_extension_point_gets_called(self):
plugin = p.get_plugin('test_rdf_harvester')
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET, self.rdf_mock_url)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD, self.rdf_mock_url, status=405)
harvest_source = self._create_harvest_source(self.rdf_mock_url)
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['after_download'], 1)
def test_ipipe_notified(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
res_id = self._test_resource(url)['id']
# celery.send_task doesn't respect CELERY_ALWAYS_EAGER
res = update_resource.apply_async(args=[self.config, res_id, 'queue1'])
res.get()
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') == None
assert params.get('resource_id') == res_id
def setup_class(cls):
model.repo.rebuild_db()
plugins.load("test_resource_preview", "test_json_resource_preview")
cls.plugin = plugins.get_plugin("test_resource_preview")
create_test_data.CreateTestData.create()
cls.package = model.Package.get("annakarenina")
cls.resource = cls.package.resources[0]
cls.url = h.url_for("resource.read",
id=cls.package.name,
resource_id=cls.resource.id)
cls.preview_url = h.url_for(
"resource.datapreview",
id=cls.package.id,
resource_id=cls.resource.id,
)
def test_harvest_after_download_empty_content_stops_gather_stage(self):
plugin = p.get_plugin('test_rdf_harvester')
source_url = 'http://return.empty.content'
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET,
source_url,
body='return.empty.content',
content_type=self.rdf_content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD,
source_url,
status=405,
content_type=self.rdf_content_type)
harvest_source = self._create_harvest_source(source_url)
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['after_download'], 1)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Check that the file was requested
assert ('return.empty.content'
in httpretty.last_request().headers['host'])
# Get the harvest source with the udpated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_(last_job_status['status'], 'Finished')
# We would expect two datasets created, so if no stats we assume the
# gather stage was stopped
eq_(last_job_status['stats'], {})
def test_after_create_plugin_hook(self):
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
with self.app.flask_app.test_request_context():
offset = url_for(controller='package', action='new')
res = self.app.get(offset, extra_environ=self.extra_environ_tester)
new_name = u'plugged2'
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = new_name
res = fv.submit('save', extra_environ=self.extra_environ_tester)
# get redirected ...
assert plugin.calls['after_update'] == 0, plugin.calls
assert plugin.calls['after_create'] == 1, plugin.calls
assert plugin.id_in_dict
plugins.unload('test_package_controller_plugin')
def test_harvest_after_download_extension_point_gets_called(self):
plugin = p.get_plugin('test_rdf_harvester')
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET, self.mock_url)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD, self.mock_url,
status=405)
harvest_source = self._create_harvest_source()
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['after_download'], 1)
def test_edit_plugin_hook(self):
# just the absolute basics
try:
plugins.load('test_package_controller_plugin')
plugin = plugins.get_plugin('test_package_controller_plugin')
res = self.app.get(self.offset, extra_environ=self.extra_environ_admin)
new_name = u'new-name'
new_title = u'New Title'
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = new_name
fv[prefix + 'title'] = new_title
res = fv.submit('save', extra_environ=self.extra_environ_admin)
# get redirected ...
assert plugin.calls['edit'] == 1, plugin.calls
plugins.unload('test_package_controller_plugin')
finally:
self._reset_data()
def test_hook(self, app):
res = factories.Resource()
plugin = plugins.get_plugin("test_resource_view")
plugin.calls.clear()
url = h.url_for("resource.read",
id=res["package_id"],
resource_id=res["id"])
result = app.get(url)
assert "There are no views created" in result
# no preview for type "ümlaut", should not fail
res["format"] = u"ümlaut"
call_action("resource_update", **res)
result = app.get(url, status=200)
assert "There are no views created" in result
res["format"] = "mock"
call_action("resource_update", **res)
assert plugin.calls["can_view"] == 2
result = app.get(url)
assert 'data-module="data-viewer"' in result.body
assert "<iframe" in result.body
views = call_action("resource_view_list", id=res["id"])
assert len(views) == 1
assert views[0]["view_type"] == "test_resource_view"
view_url = h.url_for("resource.view",
id=res["package_id"],
resource_id=res["id"],
view_id=views[0]["id"])
result = app.get(view_url)
assert plugin.calls["setup_template_variables"] == 1
assert plugin.calls["view_template"] == 1
assert "mock-preview" in result
assert "mock-preview.js" in result
def test_package_show(self, client):
url = client + '/?status=200&content=test&content-type=csv'
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
pkg_dict = {
'name': 'test-package-api',
'resources': [{
'url': url,
'format': 'TXT',
'description': 'Test'
}]
}
pkg = factories.Dataset(**pkg_dict)
update_package(pkg['id'])
result = helpers.call_action("package_show", id=pkg["id"])
print(result)
assert 'archiver' in result.keys()
def test_harvest_after_download_empty_content_stops_gather_stage(self):
plugin = p.get_plugin('test_rdf_harvester')
source_url = 'http://return.empty.content'
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET, source_url,
body='return.empty.content',
content_type=self.rdf_content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD, source_url,
status=405,
content_type=self.rdf_content_type)
harvest_source = self._create_harvest_source(source_url)
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['after_download'], 1)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Check that the file was requested
assert ('return.empty.content'
in httpretty.last_request().headers['host'])
# Get the harvest source with the udpated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_(last_job_status['status'], 'Finished')
# We would expect two datasets created, so if no stats we assume the
# gather stage was stopped
eq_(last_job_status['stats'], {})
def _resolve_alembic_config(plugin):
if plugin:
plugin_obj = p.get_plugin(plugin)
if plugin_obj is None:
tk.error_shout(u"Plugin '{}' cannot be loaded.".format(plugin))
raise click.Abort()
plugin_dir = os.path.dirname(inspect.getsourcefile(type(plugin_obj)))
# if there is `plugin` folder instead of single_file, find
# plugin's parent dir
ckanext_idx = plugin_dir.rfind(u"/ckanext/") + 9
idx = plugin_dir.find(u"/", ckanext_idx)
if ~idx:
plugin_dir = plugin_dir[:idx]
migration_dir = os.path.join(plugin_dir, u"migration", plugin)
else:
import ckan.migration as _cm
migration_dir = os.path.dirname(_cm.__file__)
return os.path.join(migration_dir, u"alembic.ini")
def get_workspaces(self):
q = request.params.get('incomplete', '')
limit = request.params.get('limit', '10')
plugin = get_plugin('wirecloud_view')
# Create a OAuth2 Session
token = toolkit.c.usertoken
oauth = OAuth2Session(self.client_id, token=token)
# Request workspaces
wirecloud_response = oauth.get(urljoin(plugin.wirecloud_url, "api/search") + "?namespace=workspace&q=" + quote_plus(q) + "&maxresults=" + quote_plus(limit))
dashboards = wirecloud_response.json()['results']
response.headers[b'Content-Type'] = b"application/json"
return json.dumps({
"ResultSet": {
"Result": [{"Name": "%s/%s" % (dashboard['owner'], dashboard['name'])} for dashboard in dashboards]
}
}).encode("utf-8")
def test_mapper_plugin_fired_on_delete():
plugin = plugins.get_plugin("mapper_plugin")
factories.Dataset(name="testpkg")
plugin.calls = []
# remove this data
user = factories.User()
context = {"user": user["name"]}
logic.get_action("package_delete")(context, {"id": "testpkg"})
# state=deleted doesn't trigger before_delete()
assert plugin.calls == []
from ckan import model
# purging the package does trigger before_delete()
model.Package.get("testpkg").purge()
model.Session.commit()
model.Session.remove()
assert plugin.calls == [
("before_delete", "testpkg"),
("after_delete", "testpkg"),
]
def test_harvest_before_download_null_url_stops_gather_stage(self):
plugin = p.get_plugin('test_rdf_harvester')
source_url = 'http://return.none'
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET,
source_url,
body=self.rdf_content,
content_type=self.rdf_content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD,
source_url,
status=405,
content_type=self.rdf_content_type)
harvest_source = self._create_harvest_source(source_url)
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['before_download'], 1)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Check that the file was not requested
assert 'return.none' not in httpretty.last_request().headers['host']
# Get the harvest source with the udpated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_(last_job_status['status'], 'Finished')
eq_(last_job_status['stats']['added'], 0)
def test_harvest_after_download_errors_get_stored(self):
plugin = p.get_plugin('test_rdf_harvester')
source_url = 'http://return.content.errors'
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET,
source_url,
body='return.errors',
content_type=self.rdf_content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD,
source_url,
status=405,
content_type=self.rdf_content_type)
harvest_source = self._create_harvest_source(source_url)
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['after_download'], 1)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Check that the file was requested
assert ('return.content.errors'
in httpretty.last_request().headers['host'])
# Get the harvest source with the udpated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_('Error 1', last_job_status['gather_error_summary'][0][0])
eq_('Error 2', last_job_status['gather_error_summary'][1][0])
def test_ipipe_notified_dataset(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
pkg = self._test_package(url)
update_package(self.config, pkg['id'], 'queue1')
assert len(testipipe.calls) == 2, len(testipipe.calls)
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == pkg['resources'][0]['id']
operation, queue, params = testipipe.calls[1]
assert operation == 'package-archived'
assert queue == 'queue1'
assert params.get('package_id') == pkg['id']
assert params.get('resource_id') is None
def test_ipipe_notified_dataset(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
pkg = self._test_package(url)
update_package(self.config, pkg['id'], 'queue1')
assert len(testipipe.calls) == 2, len(testipipe.calls)
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == pkg['resources'][0]['id']
operation, queue, params = testipipe.calls[1]
assert operation == 'package-archived'
assert queue == 'queue1'
assert params.get('package_id') == pkg['id']
assert params.get('resource_id') is None
def test_ipipe_notified_dataset(self, client):
url = client + '/?status=200&content=test&content-type=csv'
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
pkg = self._test_package(url)
update_package(pkg['id'], 'queue1')
assert len(testipipe.calls) == 2, len(testipipe.calls)
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == pkg['resources'][0]['id']
operation, queue, params = testipipe.calls[1]
assert operation == 'package-archived'
assert queue == 'queue1'
assert params.get('package_id') == pkg['id']
assert params.get('resource_id') is None
def test_harvest_after_download_errors_get_stored(self):
plugin = p.get_plugin('test_rdf_harvester')
source_url = 'http://return.content.errors'
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET, source_url,
body='return.errors',
content_type=self.rdf_content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD, source_url,
status=405,
content_type=self.rdf_content_type)
harvest_source = self._create_harvest_source(source_url)
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['after_download'], 1)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Check that the file was requested
assert ('return.content.errors'
in httpretty.last_request().headers['host'])
# Get the harvest source with the udpated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_('Error 1', last_job_status['gather_error_summary'][0][0])
eq_('Error 2', last_job_status['gather_error_summary'][1][0])
def test_harvest_before_download_null_url_stops_gather_stage(self):
plugin = p.get_plugin('test_rdf_harvester')
source_url = 'http://return.none'
# Mock the GET request to get the file
httpretty.register_uri(httpretty.GET, source_url,
body=self.rdf_content,
content_type=self.rdf_content_type)
# The harvester will try to do a HEAD request first so we need to mock
# this as well
httpretty.register_uri(httpretty.HEAD, source_url,
status=405,
content_type=self.rdf_content_type)
harvest_source = self._create_harvest_source(source_url)
self._create_harvest_job(harvest_source['id'])
self._run_jobs(harvest_source['id'])
self._gather_queue(1)
eq_(plugin.calls['before_download'], 1)
# Run the jobs to mark the previous one as Finished
self._run_jobs()
# Check that the file was not requested
assert 'return.none' not in httpretty.last_request().headers['host']
# Get the harvest source with the udpated status
harvest_source = h.call_action('harvest_source_show',
id=harvest_source['id'])
last_job_status = harvest_source['status']['last_job']
eq_(last_job_status['status'], 'Finished')
eq_(last_job_status['stats']['added'], 0)
def test_ipipe_notified_dataset(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
pkg = self._test_package(url)
# celery.send_task doesn't respect CELERY_ALWAYS_EAGER
res = update_package.apply_async(args=[self.config, pkg['id'], 'queue1'])
res.get()
assert len(testipipe.calls) == 2, len(testipipe.calls)
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') == None
assert params.get('resource_id') == pkg['resources'][0]['id']
operation, queue, params = testipipe.calls[1]
assert operation == 'package-archived'
assert queue == 'queue1'
assert params.get('package_id') == pkg['id']
assert params.get('resource_id') == None
def test_plugin_loading_order(self):
"""
Check that plugins are loaded in the order specified in the config
"""
config_plugins = config['ckan.plugins']
config['ckan.plugins'] = 'test_observer_plugin mapper_plugin mapper_plugin2'
plugins.load_all(config)
observerplugin = plugins.get_plugin('test_observer_plugin')
expected_order = _make_calls(plugins.get_plugin('mapper_plugin'),
plugins.get_plugin('mapper_plugin2'))
assert observerplugin.before_load.calls[:2] == expected_order
expected_order = _make_calls(plugins.get_plugin('test_observer_plugin'),
plugins.get_plugin('mapper_plugin'),
plugins.get_plugin('mapper_plugin2'))
assert observerplugin.after_load.calls[:3] == expected_order
config['ckan.plugins'] = 'test_observer_plugin mapper_plugin2 mapper_plugin'
plugins.load_all(config)
expected_order = _make_calls(plugins.get_plugin('mapper_plugin2'),
plugins.get_plugin('mapper_plugin'))
assert observerplugin.before_load.calls[:2] == expected_order
expected_order = _make_calls(plugins.get_plugin('test_observer_plugin'),
plugins.get_plugin('mapper_plugin2'),
plugins.get_plugin('mapper_plugin'))
assert observerplugin.after_load.calls[:3] == expected_order
# cleanup
config['ckan.plugins'] = config_plugins
plugins.load_all(config)
