def test_code_expired(self):
dataset = factories.Dataset()
subscription = SubscriptionLowLevel(
object_id=dataset['id'],
object_type='dataset',
email='*****@*****.**',
frequency=subscribe_model.Frequency.IMMEDIATE.value,
verification_code='the_code',
verification_code_expires=datetime.datetime.now() -
datetime.timedelta(hours=1) # in the past
)
with assert_raises(ValidationError) as cm:
subscription = helpers.call_action(
"subscribe_verify",
{},
code='the_code',
)
assert_in('That validation code has expired',
str(cm.exception.error_dict))
subscription = subscribe_model.Subscription.get(subscription['id'])
eq(subscription.verified, False)
def test_sysadmins_can_delete_any_resource(self):
owner_org = factories.Organization()
dataset = factories.Dataset(owner_org=owner_org['id'])
resource = factories.Resource(package_id=dataset['id'])
sysadmin = factories.Sysadmin()
app = helpers._get_test_app()
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
response = app.post(
url_for(controller='package',
action='resource_delete',
id=dataset['name'],
resource_id=resource['id']),
extra_environ=env,
)
response = response.follow()
assert_equal(200, response.status_int)
response.mustcontain('This dataset has no data')
assert_raises(p.toolkit.ObjectNotFound,
helpers.call_action,
'resource_show',
id=resource['id'])
def test_not_authorized_if_user_has_no_permissions_on_dataset_4(self):
org = factories.Organization()
user = factories.User()
member = {"username": user["name"], "role": "admin", "id": org["id"]}
helpers.call_action("organization_member_create", **member)
user_2 = factories.User()
dataset = factories.Dataset(user=user, owner_org=org["id"])
resource = {
"package_id": dataset["id"],
"title": "Resource",
"url": "http://test",
"format": "csv",
}
context = {"user": user_2["name"], "model": model}
with pytest.raises(logic.NotAuthorized):
helpers.call_auth("resource_create", context=context, **resource)
def test_trash_purge_custom_ds_type(self, app, sysadmin_env):
"""Posting the trash view with 'deleted' datasets, purges the
datasets."""
factories.Dataset(state="deleted", type="custom_dataset")
# how many datasets before purge
pkgs_before_purge = model.Session.query(model.Package).count()
assert pkgs_before_purge == 1
trash_url = url_for("admin.trash")
response = app.post(
trash_url,
data={"action": "package"},
extra_environ=sysadmin_env,
status=200
)
# check for flash success msg
assert "datasets have been purged" in response.body
# how many datasets after purge
pkgs_after_purge = model.Session.query(model.Package).count()
assert pkgs_after_purge == 0
def test_resource_delete_editor(self):
'''Normally organization admins can delete resources
Our plugin prevents this by blocking delete organization.
Ensure the delete button is not displayed (as only resource delete
is checked for showing this)
'''
user = factories.User()
owner_org = factories.Organization(users=[{
'name': user['id'],
'capacity': 'admin'
}])
dataset = factories.Dataset(owner_org=owner_org['id'])
resource = factories.Resource(package_id=dataset['id'])
with assert_raises(logic.NotAuthorized) as e:
logic.check_access('resource_delete', {'user': user['name']},
{'id': resource['id']})
assert_equal(
e.exception.message,
'User %s not authorized to delete resource %s' %
(user['name'], resource['id']))
def setup(self):
super(TestCommentModeration, self).setup()
self.user = factories.User()
self.organization = factories.Organization(user=self.user)
self.dataset = factories.Dataset(user=self.user,
owner_org=self.organization['name'])
self.issue = issue_factories.Issue(user=self.user,
user_id=self.user['id'],
dataset_id=self.dataset['id'])
self.comment = issue_factories.IssueComment(
dataset_id=self.dataset['id'],
issue_number=self.issue['number'],
comment='this is a comment',
)
self.comment2 = issue_factories.IssueComment(
dataset_id=self.dataset['id'],
issue_number=self.issue['number'],
comment='this should not be shown',
)
def test_collaborators_can_delete_resources(self, role):
org1 = factories.Organization()
dataset = factories.Dataset(owner_org=org1["id"])
resource = factories.Resource(package_id=dataset["id"])
user = factories.User()
helpers.call_action(
"package_collaborator_create",
id=dataset["id"],
user_id=user["id"],
capacity=role,
)
context = {
"user": user["name"],
"ignore_auth": False,
}
helpers.call_action("resource_delete",
context=context,
id=resource["id"])
def test_showcase_create_with_existing_name(self):
'''
Calling showcase create with an existing name raises ValidationError.
'''
sysadmin = factories.Sysadmin()
context = {'user': sysadmin['name']}
factories.Dataset(type='showcase', name='my-showcase')
# a single showcases exist.
nosetools.assert_equal(
model.Session.query(Package).filter(
Package.type == 'showcase').count(), 1)
nosetools.assert_raises(toolkit.ValidationError,
helpers.call_action,
'ckanext_showcase_create',
context=context,
name='my-showcase')
# still only one showcase exists.
nosetools.assert_equal(
model.Session.query(Package).filter(
Package.type == 'showcase').count(), 1)
def _test_resource(self,
url='anything',
format='TXT',
archived=True,
cached=True,
license_id='uk-ogl'):
pkg = {
'license_id': license_id,
'resources': [{
'url': url,
'format': format,
'description': 'Test'
}]
}
pkg = ckan_factories.Dataset(**pkg)
res_id = pkg['resources'][0]['id']
if archived:
archival = Archival.create(res_id)
archival.cache_filepath = __file__ if cached else None # just needs to exist
archival.updated = TODAY
model.Session.add(archival)
model.Session.commit()
return model.Resource.get(res_id)
def test_trash_purge_all_no_remove_deleted_packages(self, current_user, app):
"""Posting the trash view with 'deleted' entities and
purge all button purges everything"""
factories.Dataset(state="deleted", type="custom_dataset")
factories.Group(state="deleted")
factories.Organization(state="deleted")
factories.Organization(state="deleted", type="custom_org")
factories.Organization()
mock_current_user(current_user)
# how many entities before purge
pkgs_before_purge = model.Session.query(model.Package).count()
orgs_and_grps_before_purge = model.Session.query(model.Group).count()
assert pkgs_before_purge + orgs_and_grps_before_purge == 5
trash_url = url_for("admin.trash")
response = app.post(trash_url, data={"action": "all"}, status=200)
# check for flash success msg
assert "Massive purge complete" in response
# how many entities after purge
pkgs_after_purge = model.Session.query(model.Package).count()
orgs_and_grps_after_purge = model.Session.query(model.Group).count()
assert pkgs_after_purge + orgs_and_grps_after_purge == 1
def test_package_dictize_group(self):
group = factories.Group(title="Test Group Dictize")
dataset = factories.Dataset(groups=[{"name": group["name"]}])
dataset_obj = model.Package.get(dataset["id"])
context = {"model": model, "session": model.Session}
result = model_dictize.package_dictize(dataset_obj, context)
assert_equal_for_keys(result["groups"][0], group, "name")
expected_dict = {
u"approval_status": group["approval_status"],
u"capacity": "public",
u"description": group["description"],
"display_name": group["display_name"],
"image_display_url": group["image_display_url"],
u"image_url": group["image_url"],
u"is_organization": group["is_organization"],
u"name": group["name"],
u"state": group["state"],
u"title": group["title"],
u"type": group["type"],
}
self.assert_equals_expected(expected_dict, result["groups"][0])
def test_harvest_source_job_history_clear(self):
# prepare
source = factories.HarvestSourceObj(**SOURCE_DICT.copy())
job = factories.HarvestJobObj(source=source)
dataset = ckan_factories.Dataset()
object_ = factories.HarvestObjectObj(job=job, source=source,
package_id=dataset['id'])
# execute
context = {'model': model, 'session': model.Session,
'ignore_auth': True, 'user': ''}
result = toolkit.get_action('harvest_source_job_history_clear')(
context, {'id': source.id})
# verify
assert_equal(result, {'id': source.id})
source = harvest_model.HarvestSource.get(source.id)
assert source
assert_equal(harvest_model.HarvestJob.get(job.id), None)
assert_equal(harvest_model.HarvestObject.get(object_.id), None)
dataset_from_db = model.Package.get(dataset['id'])
assert dataset_from_db, 'is None'
assert_equal(dataset_from_db.id, dataset['id'])
def test_move_to_published_stage_email(self):
user = factories.User()
sysadmin = factories.Sysadmin()
pkg = factories.Dataset(user=user)
revision = th.call_action('create_dataset_revision',
{'user': user['name']},
id=pkg['id'])
with mock.patch('ckan.lib.mailer.mail_recipient') as mock_mail:
th.call_action('move_to_next_stage', {'user': user['name']},
id=revision['id'])
th.call_action('move_to_next_stage', {'user': sysadmin['name']},
id=revision['id'])
nt.assert_true(mock_mail.called)
args = mock_mail.call_args[0]
subject = args[2]
nt.assert_in('approval', subject)
text = args[3]
nt.assert_in(revision['title'], text)
nt.assert_in('revision', text)
nt.assert_in(pkg['id'], text)
nt.assert_in(sysadmin['name'], text)
def test_migration(self):
# Test that the migration correctly takes the package_revision (etc)
# tables and populates the Activity.data, i.e. it does it the same as
# if you made a change to the dataset with the current version of CKAN
# and the Activity was created by activity_stream_item().
dataset = factories.Dataset(resources=[{
u'url': u'http://example.com/a.csv',
u'format': u'csv'
}])
activity = package_activity_list(dataset['id'], 0, 0)[0]
activity_data_as_it_should_be = copy.deepcopy(activity.data)
# Remove 'activity.data.package' to provoke the migration to regenerate
# it from package_revision (etc)
activity = model.Activity.get(activity.id)
del activity.data['package']
model.repo.commit_and_remove()
with PackageDictizeMonkeyPatch():
migrate_dataset(dataset['name'], {})
activity_data_migrated = \
package_activity_list(dataset['id'], 0, 0)[0].data
eq_(activity_data_as_it_should_be, activity_data_migrated)
def test_catalog_ttl(self):
for i in xrange(4):
factories.Dataset()
url = url_for('dcat_catalog', _format='ttl')
app = self._get_test_app()
response = app.get(url)
eq_(response.headers['Content-Type'], 'text/turtle')
content = response.body
# Parse the contents to check it's an actual serialization
p = RDFParser()
p.parse(content, _format='turtle')
dcat_datasets = [d for d in p.datasets()]
eq_(len(dcat_datasets), 4)
def test_add_views_to_dataset_resources(self):
# New resources have no views
dataset_dict = factories.Dataset(resources=[
{
'url': 'http://some.image.png',
'format': 'png',
'name': 'Image 1',
},
{
'url': 'http://some.image.png',
'format': 'png',
'name': 'Image 2',
},
])
context = {'user': helpers.call_action('get_site_user')['name']}
created_views = datapreview.add_views_to_dataset_resources(
context, dataset_dict, view_types=['image_view'])
eq_(len(created_views), 2)
eq_(created_views[0]['view_type'], 'image_view')
eq_(created_views[1]['view_type'], 'image_view')
def test_add_default_views_to_resource(self):
# New resources have no views
dataset_dict = factories.Dataset()
resource_dict = factories.Resource(
package_id=dataset_dict['id'],
url='http://some.image.png',
format='png',
)
# Change default views config setting
config['ckan.views.default_views'] = 'image_view'
context = {'user': helpers.call_action('get_site_user')['name']}
created_views = helpers.call_action(
'resource_create_default_resource_views',
context,
resource=resource_dict,
package=dataset_dict)
assert_equals(len(created_views), 1)
assert_equals(created_views[0]['view_type'], 'image_view')
def test_showcase_new_redirects_to_manage_datasets(self):
'''Creating a new showcase redirects to the manage datasets form.'''
app = self._get_test_app()
sysadmin = factories.Sysadmin()
# need a dataset for the 'bulk_action.showcase_add' button to show
factories.Dataset()
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
response = app.get(
url=url_for(controller='ckanext.showcase.controller:ShowcaseController', action='new'),
extra_environ=env,
)
# create showcase
form = response.forms['dataset-edit']
form['name'] = u'my-showcase'
create_response = submit_and_follow(app, form, env, 'save')
# Unique to manage_datasets page
nosetools.assert_true('bulk_action.showcase_add' in create_response)
# Requested page is the manage_datasets url.
nosetools.assert_equal(url_for(controller='ckanext.showcase.controller:ShowcaseController',
action='manage_datasets', id='my-showcase'), create_response.request.path)
def test_basic_access(self):
"""Normally organization admins can delete resources
Our plugin prevents this by blocking delete organization.
Ensure the delete button is not displayed (as only resource delete
is checked for showing this)
"""
owner = factories.User()
access = factories.User()
owner_org = factories.Organization(
users=[{'name': owner['id'], 'capacity': 'admin'}]
)
dataset = factories.Dataset(owner_org=owner_org['id'], private=True)
resource = factories.Resource(package_id=dataset['id'])
assert logic.check_access('package_show', {'user': owner['name']}, {'id': dataset['id']})
assert logic.check_access('resource_show', {'user': owner['name']}, {'id': resource['id']})
with pytest.raises(logic.NotAuthorized):
logic.check_access('package_show', {'user': access['name']}, {'id': dataset['id']})
with pytest.raises(logic.NotAuthorized):
logic.check_access('resource_show', {'user': access['name']}, {'id': resource['id']})
def test_change_dataset(self, app):
user = factories.User()
group = factories.Group(user=user)
dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user)
self._clear_activities()
dataset["title"] = "Dataset with changed title"
helpers.call_action(
"package_update", context={"user": user["name"]}, **dataset
)
url = url_for("group.activity", id=group["id"])
response = app.get(url)
assert (
'<a href="/user/{}">Mr. Test User'.format(user["name"]) in response
)
assert "updated the dataset" in response
assert (
'<a href="/dataset/{}">Dataset with changed title'.format(
dataset["id"]
)
in response
)
def test_package_delete_removes_collaborations():
user = factories.User()
dataset = factories.Dataset()
helpers.call_action(
"package_collaborator_create",
id=dataset["id"],
user_id=user["id"],
capacity="editor",
)
assert (len(
helpers.call_action("package_collaborator_list_for_user",
id=user["id"])) == 1)
context = {}
params = {u"id": dataset[u"id"]}
helpers.call_action(u"package_delete", context, **params)
assert (len(
helpers.call_action("package_collaborator_list_for_user",
id=user["id"])) == 0)
def test_validation_fails_on_upload(self, mock_open, app):
invalid_file = StringIO.StringIO()
invalid_file.write(INVALID_CSV)
mock_upload = MockFieldStorage(invalid_file, 'invalid.csv')
dataset = factories.Dataset()
invalid_stream = io.BufferedReader(io.BytesIO(INVALID_CSV))
with mock.patch('io.open', return_value=invalid_stream):
with assert_raises(t.ValidationError) as e:
call_action('resource_create',
package_id=dataset['id'],
format='CSV',
upload=mock_upload)
assert 'validation' in e.exception.error_dict
assert 'missing-value' in str(e.exception)
assert 'Row 2 has a missing value in column 4' in str(e.exception)
def test_add_default_views_to_resource_no_dataset_passed(self):
# New resources have no views
dataset_dict = factories.Dataset()
resource_dict = factories.Resource(
package_id=dataset_dict["id"],
url="http://some.image.png",
format="png",
)
# Change default views config setting
config["ckan.views.default_views"] = "image_view"
context = {"user": helpers.call_action("get_site_user")["name"]}
created_views = helpers.call_action(
"resource_create_default_resource_views",
context,
resource=resource_dict,
)
assert len(created_views) == 1
assert created_views[0]["view_type"] == "image_view"
def test_validation_passes_on_upload(self, mock_open, app):
dataset = factories.Dataset(
resources=[{
'url': 'https://example.com/data.csv'
}])
valid_file = StringIO.StringIO()
valid_file.write(INVALID_CSV)
mock_upload = MockFieldStorage(valid_file, 'valid.csv')
valid_stream = io.BufferedReader(io.BytesIO(VALID_CSV))
with mock.patch('io.open', return_value=valid_stream):
resource = call_action('resource_update',
id=dataset['resources'][0]['id'],
format='CSV',
upload=mock_upload)
assert_equals(resource['validation_status'], 'success')
assert 'validation_timestamp' in resource
def test_add_views_to_dataset_resources(self):
# New resources have no views
dataset_dict = factories.Dataset(resources=[
{
"url": "http://some.image.png",
"format": "png",
"name": "Image 1",
},
{
"url": "http://some.image.png",
"format": "png",
"name": "Image 2",
},
])
context = {"user": helpers.call_action("get_site_user")["name"]}
created_views = datapreview.add_views_to_dataset_resources(
context, dataset_dict, view_types=["image_view"])
assert len(created_views) == 2
assert created_views[0]["view_type"] == "image_view"
assert created_views[1]["view_type"] == "image_view"
def test_org(self):
from ckan import model
org = ckan_factories.Organization()
subscribe_model.Subscribe.set_emails_last_sent(
subscribe_model.Frequency.IMMEDIATE.value, datetime.datetime.now())
dataset = ckan_factories.Dataset(owner_org=org['id'])
activity = model.Session.query(model.Activity) \
.filter_by(object_id=dataset['id']).first()
subscription_activities = {
factories.Subscription(organization_id=org['id'],
return_object=True): [activity]
}
notifications = dictize_notifications(subscription_activities)
email = get_notification_email_contents(code='the-code',
email='*****@*****.**',
notifications=notifications)
# Check we link to the dataset, not just the org
assert_in('http://test.ckan.net/dataset/{}'.format(dataset['name']),
email[1])
assert_in(
'<a href="http://test.ckan.net/dataset/{}">Test Dataset</a>'.
format(dataset['name']), email[2])
def test_not_authorized_if_user_has_no_permissions_on_dataset_2(self):
org = factories.Organization()
user = factories.User()
member = {"username": user["name"], "role": "admin", "id": org["id"]}
helpers.call_action("organization_member_create", **member)
user_2 = factories.User()
dataset = factories.Dataset(owner_org=org["id"])
resource = factories.Resource(package_id=dataset["id"])
context = {"user": user_2["name"], "model": model}
with pytest.raises(logic.NotAuthorized):
helpers.call_auth(
"resource_create_default_resource_views",
context=context,
resource=resource,
)
def test_dataset_autocomplete_name(self):
dataset = factories.Dataset(name='rivers')
url = url_for(controller='api', action='dataset_autocomplete', ver='/2')
assert_equal(url, '/api/2/util/dataset/autocomplete')
app = self._get_test_app()
response = app.get(
url=url,
params={
'incomplete': u'rive',
},
status=200,
)
results = json.loads(response.body)
assert_equal(results, {"ResultSet": {"Result": [{
'match_field': 'name',
"name": "rivers",
'match_displayed': 'rivers',
'title': dataset['title'],
}]}})
assert_equal(response.headers['Content-Type'],
'application/json;charset=utf-8')
def test_delete(self, app):
self.user = factories.User()
self.user_env = {"REMOTE_USER": six.ensure_str(self.user["name"])}
self.organization = factories.Organization(user=self.user)
datasets = [
factories.Dataset(owner_org=self.organization["id"], private=True)
for i in range(0, 5)
]
form = {'dataset_' + d["id"]: "on" for d in datasets}
form["bulk_action.delete"] = "delete"
app.post(
url=url_for(
"organization.bulk_process", id=self.organization["id"]
),
extra_environ=self.user_env,
data=form
)
for dataset in datasets:
d = helpers.call_action("package_show", id=dataset["id"])
assert d["state"] == "deleted"
def test_worker_datastore_access(self):
'''
Test DataStore access from within a worker.
'''
pkg = factories.Dataset()
data = {
'resource': {
'package_id': pkg['id'],
},
'fields': [{
'id': 'value',
'type': 'int'
}],
}
table = helpers.call_action('datastore_create', **data)
res_id = table['resource_id']
for i in range(3):
self.enqueue(datastore_job, args=[res_id, i])
jobs.Worker().work(burst=True)
# Aside from ensuring that the job succeeded, this also checks
# that accessing the Datastore still works in the main process.
result = helpers.call_action('datastore_search', resource_id=res_id)
assert_equal([0, 1, 2], [r['value'] for r in result['records']])
