def test_providing_res_with_url_calls_datapusher_correctly(self):
config['datapusher.url'] = 'http://datapusher.ckan.org'
responses.add(
responses.POST,
'http://datapusher.ckan.org/job',
content_type='application/json',
body=json.dumps({'job_id': 'foo', 'job_key': 'bar'}))
responses.add_passthru(config['solr_url'])
package = model.Package.get('annakarenina')
tests.call_action_api(
self.app,
'datastore_create',
apikey=self.sysadmin_user.apikey,
resource=dict(package_id=package.id, url='demo.ckan.org')
)
assert len(package.resources) == 4, len(package.resources)
resource = package.resources[3]
data = json.loads(responses.calls[-1].request.body)
assert data['metadata']['resource_id'] == resource.id, data
assert not data['metadata'].get('ignore_hash'), data
assert data['result_url'].endswith('/action/datapusher_hook'), data
assert data['result_url'].startswith('http://'), data
def test_resource_with_0_views(self):
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
resource = self._create_resource(app, package, apikey)
# The package_show() API should return 0 recent views and 0 total
# views for the unviewed resource.
package = tests.call_action_api(app, 'package_show',
id=package['name'],
include_tracking=True)
assert len(package['resources']) == 1
resource = package['resources'][0]
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 0, ("A resource that has not "
"been viewed should have 0 "
"recent views")
assert tracking_summary['total'] == 0, ("A resource that has not "
"been viewed should have 0 "
"total views")
# The resource_show() API should return 0 recent views and 0 total
# views for the unviewed resource.
resource = tests.call_action_api(app, 'resource_show',
id=resource['id'],
include_tracking=True)
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 0, ("A resource that has not "
"been viewed should have 0 "
"recent views")
assert tracking_summary['total'] == 0, ("A resource that has not "
"been viewed should have 0 "
"total views")
def test_03_create_private_package(self):
# Make an organization, because private datasets must belong to one.
organization = tests.call_action_api(self.app, 'organization_create',
name='test_org',
apikey=self.sysadmin_user.apikey)
# Create a dataset without specifying visibility
package_dict = {
'extras': [{'key': u'original media','value': u'"book"'}],
'license_id': u'other-open',
'maintainer_email': None,
'name': u'annakarenina_vis',
'notes': u'Some test now',
'resources': [{'alt_url': u'alt123',
'description': u'Full text.',
'extras': {u'alt_url': u'alt123', u'size': u'123'},
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'url': u'http://datahub.io/download/'},
{'alt_url': u'alt345',
'description': u'Index of the novel',
'extras': {u'alt_url': u'alt345', u'size': u'345'},
'format': u'JSON',
'hash': u'def456',
'position': 1,
'url': u'http://datahub.io/index.json'}],
'tags': [{'name': u'russian'}, {'name': u'tolstoy'}],
'title': u'A Novel By Tolstoy',
'url': u'http://datahub.io',
'owner_org': organization['id'],
'version': u'0.7a',
}
package_created = tests.call_action_api(self.app, 'package_create',
apikey=self.sysadmin_user.apikey,
**package_dict)
assert package_created['private'] is False
# Create a new one, explicitly saying it is public
package_dict['name'] = u'annakareninanew_vis_public'
package_dict['private'] = False
package_created_public = tests.call_action_api(self.app,
'package_create',
apikey=self.sysadmin_user.apikey,
**package_dict)
assert package_created_public['private'] is False
# Create a new one, explicitly saying it is private
package_dict['name'] = u'annakareninanew_vis_private'
package_dict['private'] = True
package_created_private = tests.call_action_api(self.app,
'package_create',
apikey=self.sysadmin_user.apikey,
**package_dict)
assert package_created_private['private'] is True
def test_user_cannot_create_group_when_config_is_False(self):
app = self._get_app(users_can_create_groups=False)
user = factories.User()
tests.call_action_api(app,
'group_create',
name='test-group',
apikey=user['apikey'],
status=403)
def test_new_org_request(self):
tests.call_action_api(self.app,
'hdx_send_new_org_request',
title='Org Title',
description='Org Description',
org_url='http://test-org.com/',
your_name='Some Name',
your_email='*****@*****.**',
status=403)
def test_hdx_solr_additions(self):
testsysadmin = model.User.by_name('testsysadmin')
legacy_tests.call_action_api(self.app, 'group_create', name="col", title="Colombia", apikey=testsysadmin.apikey,
status=200)
context = {'ignore_auth': True,
'model': model, 'session': model.Session, 'user': '******'}
try:
self._get_action('organization_create')(context, organization)
except Exception, ex:
log.error(ex)
def test_04_no_repeat_email_notifications(self):
'''Test that a user does not get a second email notification for the
same new activity.
'''
# TODO: Assert that Sara has some new activities and has already had
# an email about them.
tests.call_action_api(self.app, 'send_email_notifications',
apikey=self.testsysadmin['apikey'])
assert len(self.get_smtp_messages()) == 0
def test_04_no_repeat_email_notifications(self):
'''Test that a user does not get a second email notification for the
same new activity.
'''
# TODO: Assert that Sara has some new activities and has already had
# an email about them.
tests.call_action_api(self.app, 'send_email_notifications',
apikey=self.testsysadmin['apikey'])
assert len(self.get_smtp_messages()) == 0
def test_editor_request_for_org(self):
tests.call_action_api(self.app,
'hdx_send_editor_request_for_org',
display_name='User Name',
name='username',
email='*****@*****.**',
organization='Org Name',
message='Some message',
admins=[],
status=403)
def test_group_create_with_no_curators_group(self):
'''Test that group_create doesn't crash when there's no curators group.
'''
sysadmin = factories.Sysadmin()
# Make sure there's no curators group.
assert 'curators' not in tests.call_action_api(self.app, 'group_list')
# Make our sysadmin user create a group. CKAN should not crash.
tests.call_action_api(self.app, 'group_create', name='test-group',
apikey=sysadmin['apikey'])
def test_package_update_remove_org_error(self):
app = helpers._get_test_app()
org = call_action_api(app, 'organization_create',
apikey=self.sysadmin_user.apikey, name='myorganization')
package = call_action_api(app, 'package_create',
apikey=self.sysadmin_user.apikey, name='foobarbaz', owner_org=org['id'])
assert package['owner_org']
package['owner_org'] = ''
res = call_action_api(app, 'package_update',
apikey=self.sysadmin_user.apikey, **package)
assert not res['owner_org'], res['owner_org']
def test_make_package_with_no_organization_private(self):
'''Test that private packages with no organization cannot be created
by package_update.
'''
testsysadmin = model.User.by_name('testsysadmin')
package = tests.call_action_api(self.app, 'package_create',
name='test_2', private=False, apikey=testsysadmin.apikey)
package['private'] = True
result = tests.call_action_api(self.app, 'package_update',
apikey=testsysadmin.apikey, status=409, **package)
assert result == {'__type': 'Validation Error',
'private': ["Datasets with no organization can't be private."]}
def test_make_package_with_no_organization_private(self):
'''Test that private packages with no organization cannot be created
by package_update.
'''
testsysadmin = model.User.by_name('testsysadmin')
package = tests.call_action_api(self.app, 'package_create',
name='test_2', private=False, apikey=testsysadmin.apikey)
package['private'] = True
result = tests.call_action_api(self.app, 'package_update',
apikey=testsysadmin.apikey, status=409, **package)
assert result == {'__type': 'Validation Error',
'private': ["Datasets with no organization can't be private."]}
def _call_datapusher_hook(self, user, app):
package = model.Package.get("annakarenina")
resource = package.resources[0]
context = {"user": self.sysadmin_user.name}
p.toolkit.get_action("task_status_update")(
context,
{
"entity_id": resource.id,
"entity_type": "resource",
"task_type": "datapusher",
"key": "datapusher",
"value": '{"job_id": "my_id", "job_key":"my_key"}',
"last_updated": str(datetime.datetime.now()),
"state": "pending",
},
)
data = {"status": "success", "metadata": {"resource_id": resource.id}}
auth = {"Authorization": str(user.apikey)}
res = app.post(
"/api/action/datapusher_hook",
json=data,
extra_environ=auth,
status=200,
)
res_dict = json.loads(res.body)
assert res_dict["success"] is True
task = tests.call_action_api(
app,
"task_status_show",
entity_id=resource.id,
task_type="datapusher",
key="datapusher",
)
assert task["state"] == "success", task
task = tests.call_action_api(
app,
"task_status_show",
entity_id=resource.id,
task_type="datapusher",
key="datapusher",
)
assert task["state"] == "success", task
def test_01_package_list_private(self):
tests.call_action_api(self.app, 'organization_create',
name='test_org_2',
apikey=self.sysadmin_user.apikey)
tests.call_action_api(self.app, 'package_create',
name='public_dataset',
owner_org='test_org_2',
apikey=self.sysadmin_user.apikey)
res = tests.call_action_api(self.app, 'package_list')
assert len(res) == 3
assert 'warandpeace' in res
assert 'annakarenina' in res
assert 'public_dataset' in res
tests.call_action_api(self.app, 'package_create',
name='private_dataset',
owner_org='test_org_2',
private=True,
apikey=self.sysadmin_user.apikey)
res = tests.call_action_api(self.app, 'package_list')
assert len(res) == 3
assert 'warandpeace' in res
assert 'annakarenina' in res
assert 'public_dataset' in res
assert 'private_dataset' not in res
def test_01_package_list_private(self):
tests.call_action_api(self.app, 'organization_create',
name='test_org_2',
apikey=self.sysadmin_user.apikey)
tests.call_action_api(self.app, 'package_create',
name='public_dataset',
owner_org='test_org_2',
apikey=self.sysadmin_user.apikey)
res = tests.call_action_api(self.app, 'package_list')
assert len(res) == 3
assert 'warandpeace' in res
assert 'annakarenina' in res
assert 'public_dataset' in res
tests.call_action_api(self.app, 'package_create',
name='private_dataset',
owner_org='test_org_2',
private=True,
apikey=self.sysadmin_user.apikey)
res = tests.call_action_api(self.app, 'package_list')
assert len(res) == 3
assert 'warandpeace' in res
assert 'annakarenina' in res
assert 'public_dataset' in res
assert not 'private_dataset' in res
def test_group_create_with_no_curators_group(self):
'''Test that group_create doesn't crash when there's no curators group.
'''
sysadmin = factories.Sysadmin()
# Make sure there's no curators group.
assert 'curators' not in tests.call_action_api(self.app, 'group_list')
# Make our sysadmin user create a group. CKAN should not crash.
tests.call_action_api(self.app,
'group_create',
name='test-group',
apikey=sysadmin['apikey'])
def test_package_update_remove_org_error(self):
import paste.fixture
import pylons.test
app = helpers._get_test_app()
org = call_action_api(app, 'organization_create',
apikey=self.sysadmin_user.apikey, name='myorganization')
package = call_action_api(app, 'package_create',
apikey=self.sysadmin_user.apikey, name='foobarbaz', owner_org=org['id'])
assert package['owner_org']
package['owner_org'] = ''
res = call_action_api(app, 'package_update',
apikey=self.sysadmin_user.apikey, **package)
assert not res['owner_org'], res['owner_org']
def test_delete_country(self):
testsysadmin = model.User.by_name('testsysadmin')
user = model.User.by_name('tester')
create_result = tests.call_action_api(self.app,
'group_create',
name='test_group_c',
title='Test Group C',
apikey=testsysadmin.apikey,
status=200)
tests.call_action_api(self.app,
'group_delete',
id=create_result['id'],
apikey=user.apikey,
status=403)
assert True, 'user should not be allowed to delete the group'
def test_delete_org(self):
testsysadmin = model.User.by_name('testsysadmin')
user = model.User.by_name('tester')
create_result = tests.call_action_api(self.app,
'organization_create',
name='test_org_c',
title='Test Org C',
apikey=testsysadmin.apikey,
status=200)
tests.call_action_api(self.app,
'organization_delete',
id=create_result['id'],
apikey=user.apikey,
status=403)
assert True, 'user should not be allowed to delete the org'
def test_package_update_remove_org_error(self):
import paste.fixture
import pylons.test
app = paste.fixture.TestApp(pylons.test.pylonsapp)
org = call_action_api(app, 'organization_create',
apikey=self.sysadmin_user.apikey, name='myorganization')
package = call_action_api(app, 'package_create',
apikey=self.sysadmin_user.apikey, name='foobarbaz', owner_org=org['id'])
assert package['owner_org']
package['owner_org'] = ''
res = call_action_api(app, 'package_update',
apikey=self.sysadmin_user.apikey, **package)
assert not res['owner_org'], res['owner_org']
def test_package_update_duplicate_extras_error(self):
# We need to create a package first, so that we can update it.
app = helpers._get_test_app()
package = call_action_api(app, 'package_create',
apikey=self.sysadmin_user.apikey, name='foobar')
# Posting a dataset dict to package_update containing two extras dicts
# with the same key, should return a Validation Error.
package['extras'] = [{'key': 'foo', 'value': 'bar'},
{'key': 'foo', 'value': 'gar'}]
error = call_action_api(app, 'package_update',
apikey=self.sysadmin_user.apikey, status=409, **package)
assert error['__type'] == 'Validation Error'
assert error['extras_validation'] == ['Duplicate key "foo"']
def test_resource_with_many_downloads(self):
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
resource = self._create_resource(app, package, apikey)
url = resource['url']
# Download the resource three times from different IPs.
self._post_to_tracking(app, url, type_='resource', ip='111.222.333.44')
self._post_to_tracking(app, url, type_='resource', ip='111.222.333.55')
self._post_to_tracking(app, url, type_='resource', ip='111.222.333.66')
self._update_tracking_summary()
package = tests.call_action_api(app, 'package_show', id=package['id'],
include_tracking=True)
assert len(package['resources']) == 1
resource = package['resources'][0]
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 3, (
"A resource that has been downloaded 3 times recently should have "
"3 recent downloads")
assert tracking_summary['total'] == 3, (
"A resource that has been downloaded 3 times should have 3 total "
"downloads")
tracking_summary = package['tracking_summary']
assert tracking_summary['recent'] == 0, (
"Downloading a resource should not increase the resource's "
"package's recent views")
assert tracking_summary['total'] == 0, (
"Downloading a resource should not increase the resource's "
"package's total views")
def test_package_with_one_view(self):
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
self._create_resource(app, package, apikey)
url = routes.url_for(controller='package', action='read',
id=package['name'])
self._post_to_tracking(app, url)
self._update_tracking_summary()
package = tests.call_action_api(app, 'package_show', id=package['id'],
include_tracking=True)
tracking_summary = package['tracking_summary']
assert tracking_summary['recent'] == 1, ("A package that has been "
"viewed once should have 1 "
"recent view.")
assert tracking_summary['total'] == 1, ("A package that has been "
"viewed once should have 1 "
"total view")
assert len(package['resources']) == 1
resource = package['resources'][0]
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 0, ("Viewing a package should "
"not increase the recent "
"views of the package's "
"resources")
assert tracking_summary['total'] == 0, ("Viewing a package should "
"not increase the total views "
"of the package's resources")
def test_dataset_view_count_throttling(self, app):
"""If the same user visits the same dataset multiple times on the same
day, only one view should get counted.
"""
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
self._create_resource(app, package, apikey)
url = url_for("dataset.read", id=package["name"])
# Visit the dataset three times from the same IP.
self._post_to_tracking(app, url)
self._post_to_tracking(app, url)
self._post_to_tracking(app, url)
self._update_tracking_summary()
package = tests.call_action_api(
app, "package_show", id=package["id"], include_tracking=True
)
tracking_summary = package["tracking_summary"]
assert tracking_summary["recent"] == 1, (
"Repeat dataset views should " "not add to recent views " "count"
)
assert tracking_summary["total"] == 1, (
"Repeat dataset views should " "not add to total views count"
)
def test_resource_with_many_downloads(self):
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
resource = self._create_resource(app, package, apikey)
url = resource['url']
# Download the resource three times from different IPs.
self._post_to_tracking(app, url, type_='resource', ip='111.222.333.44')
self._post_to_tracking(app, url, type_='resource', ip='111.222.333.55')
self._post_to_tracking(app, url, type_='resource', ip='111.222.333.66')
self._update_tracking_summary()
package = tests.call_action_api(app,
'package_show',
id=package['id'],
include_tracking=True)
assert len(package['resources']) == 1
resource = package['resources'][0]
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 3, (
"A resource that has been downloaded 3 times recently should have "
"3 recent downloads")
assert tracking_summary['total'] == 3, (
"A resource that has been downloaded 3 times should have 3 total "
"downloads")
tracking_summary = package['tracking_summary']
assert tracking_summary['recent'] == 0, (
"Downloading a resource should not increase the resource's "
"package's recent views")
assert tracking_summary['total'] == 0, (
"Downloading a resource should not increase the resource's "
"package's total views")
def initial_data(self, clean_db, app):
CreateTestData.create()
users = {}
tester = model.User.by_name(u"tester")
tests.call_action_api(app,
"organization_create",
name="test_org_2",
apikey=tester.apikey)
tests.call_action_api(
app,
"package_create",
name="crimeandpunishment",
owner_org="test_org_2",
apikey=tester.apikey,
)
def test_deleted_badge_appears(self):
global package
global organization
testsysadmin = model.User.by_name('testsysadmin')
dataset_name = package['name']
context = {
'model': model,
'session': model.Session,
'user': '******'
}
self._get_action('organization_create')(context, organization)
self._get_action('package_create')(context, package)
page = self._getPackagePage(dataset_name)
assert not 'Deleted' in str(
page.response
), 'Page should not have deleted badge as it was not deleted'
deleted_result = tests.call_action_api(self.app,
'package_delete',
apikey=testsysadmin.apikey,
id=dataset_name)
deleted_page = self._getPackagePage(dataset_name, testsysadmin.apikey)
# print deleted_page.response
assert 'Deleted' in str(
deleted_page.response), 'Page needs to have deleted badge'
def setup_class(cls):
model.repo.rebuild_db()
mock_mail_server.SmtpServerHarness.setup_class()
tests.CreateTestData.create()
cls.app = helpers._get_test_app()
joeadmin = model.User.get("joeadmin")
cls.joeadmin = {"id": joeadmin.id, "apikey": joeadmin.apikey}
testsysadmin = model.User.get("testsysadmin")
cls.testsysadmin = {
"id": testsysadmin.id,
"apikey": testsysadmin.apikey,
}
annafan = model.User.get("annafan")
cls.annafan = {"id": annafan.id, "apikey": annafan.apikey}
# Register a new user.
cls.sara = tests.call_action_api(
cls.app,
"user_create",
apikey=cls.testsysadmin["apikey"],
name="sara",
email="*****@*****.**",
password="******",
fullname="Sara Rollins",
activity_streams_email_notifications=True,
)
def _query_basic_user_info(self, user):
basic_info = tests.call_action_api(self.app,
'hdx_basic_user_info',
id=user.get('id'),
status=200,
apikey=user.get('apikey'))
return basic_info
def test_dataset_view_count_throttling(self):
'''If the same user visits the same dataset multiple times on the same
day, only one view should get counted.
'''
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
self._create_resource(app, package, apikey)
url = routes.url_for(controller='package', action='read',
id=package['name'])
# Visit the dataset three times from the same IP.
self._post_to_tracking(app, url)
self._post_to_tracking(app, url)
self._post_to_tracking(app, url)
self._update_tracking_summary()
package = tests.call_action_api(app, 'package_show', id=package['id'],
include_tracking=True)
tracking_summary = package['tracking_summary']
assert tracking_summary['recent'] == 1, ("Repeat dataset views should "
"not add to recent views "
"count")
assert tracking_summary['total'] == 1, ("Repeat dataset views should "
"not add to total views count")
def test_resource_download_count_throttling(self):
'''If the same user downloads the same resource multiple times on the
same day, only one view should get counted.
'''
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
resource = self._create_resource(app, package, apikey)
# Download the resource three times from the same IP.
self._post_to_tracking(app, resource['url'], type_='resource')
self._post_to_tracking(app, resource['url'], type_='resource')
self._post_to_tracking(app, resource['url'], type_='resource')
self._update_tracking_summary()
resource = tests.call_action_api(app, 'resource_show',
id=resource['id'],
include_tracking=True)
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 1, (
"Repeat resource downloads should not add to recent views count")
assert tracking_summary['total'] == 1, (
"Repeat resource downloads should not add to total views count")
def _create_package(self, app, apikey, name='look_to_windward'):
'''Create a package via the action api.'''
return tests.call_action_api(app,
'package_create',
apikey=apikey,
name=name)
def _group_create(self, group_name):
'''Return a group with some users and a dataset.'''
# Make a group with some users and a dataset.
group = tests.call_action_api(
self.app,
'group_create',
apikey=self.sysadmin.apikey,
name=group_name,
users=[{
'name': self.member['name'],
'capacity': 'member',
}, {
'name': self.editor['name'],
'capacity': 'editor',
}, {
'name': self.admin['name'],
'capacity': 'admin',
}],
packages=[{
'id': self.package['name']
}],
)
return group
def test_dataset_view_count_throttling(self):
'''If the same user visits the same dataset multiple times on the same
day, only one view should get counted.
'''
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
self._create_resource(app, package, apikey)
with app.flask_app.test_request_context():
url = url_for(controller='package',
action='read',
id=package['name'])
# Visit the dataset three times from the same IP.
self._post_to_tracking(app, url)
self._post_to_tracking(app, url)
self._post_to_tracking(app, url)
self._update_tracking_summary()
package = tests.call_action_api(app,
'package_show',
id=package['id'],
include_tracking=True)
tracking_summary = package['tracking_summary']
assert tracking_summary['recent'] == 1, ("Repeat dataset views should "
"not add to recent views "
"count")
assert tracking_summary['total'] == 1, ("Repeat dataset views should "
"not add to total views count")
def test_resource_download_count_throttling(self, app):
"""If the same user downloads the same resource multiple times on the
same day, only one view should get counted.
"""
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
resource = self._create_resource(app, package, apikey)
# Download the resource three times from the same IP.
self._post_to_tracking(app, resource["url"], type_="resource")
self._post_to_tracking(app, resource["url"], type_="resource")
self._post_to_tracking(app, resource["url"], type_="resource")
self._update_tracking_summary()
resource = tests.call_action_api(
app, "resource_show", id=resource["id"], include_tracking=True
)
tracking_summary = resource["tracking_summary"]
assert (
tracking_summary["recent"] == 1
), "Repeat resource downloads should not add to recent views count"
assert (
tracking_summary["total"] == 1
), "Repeat resource downloads should not add to total views count"
def test_resource_download_count_throttling(self):
'''If the same user downloads the same resource multiple times on the
same day, only one view should get counted.
'''
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
resource = self._create_resource(app, package, apikey)
# Download the resource three times from the same IP.
self._post_to_tracking(app, resource['url'], type_='resource')
self._post_to_tracking(app, resource['url'], type_='resource')
self._post_to_tracking(app, resource['url'], type_='resource')
self._update_tracking_summary()
resource = tests.call_action_api(app,
'resource_show',
id=resource['id'],
include_tracking=True)
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 1, (
"Repeat resource downloads should not add to recent views count")
assert tracking_summary['total'] == 1, (
"Repeat resource downloads should not add to total views count")
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
plugin = p.load('datastore')
if plugin.legacy_mode:
# make sure we undo adding the plugin
p.unload('datastore')
raise nose.SkipTest("SQL tests are not supported in legacy mode")
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
resource = cls.dataset.resources[0]
cls.data = {
'resource_id': resource.id,
'force': True,
'aliases': 'books4',
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'published'}],
'records': [{u'b\xfck': 'annakarenina',
'author': 'tolstoy',
'published': '2005-03-01',
'nested': ['b', {'moo': 'moo'}]},
{u'b\xfck': 'warandpeace',
'author': 'tolstoy',
'nested': {'a': 'b'}}]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app, 'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{u'_full_text': [u"'annakarenina'", u"'b'",
u"'moo'", u"'tolstoy'",
u"'2005'"],
u'_id': 1,
u'author': u'tolstoy',
u'b\xfck': u'annakarenina',
u'nested': [u'b', {u'moo': u'moo'}],
u'published': u'2005-03-01T00:00:00'},
{u'_full_text': [u"'tolstoy'", u"'warandpeac'",
u"'b'"],
u'_id': 2,
u'author': u'tolstoy',
u'b\xfck': u'warandpeace',
u'nested': {u'a': u'b'},
u'published': None}]
cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}]
engine = db.get_write_engine()
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def test_sorting_datasets_by_total_views(self, app, reset_index):
# FIXME: Have some datasets with different numbers of recent and total
# views, to make this a better test.
reset_index()
sysadmin_user, apikey = self._create_sysadmin(app)
self._create_package(app, apikey, name="consider_phlebas")
self._create_package(app, apikey, name="the_player_of_games")
self._create_package(app, apikey, name="use_of_weapons")
url = url_for("dataset.read", id="consider_phlebas")
self._post_to_tracking(app, url)
url = url_for("dataset.read", id="the_player_of_games")
self._post_to_tracking(app, url, ip="111.11.111.111")
self._post_to_tracking(app, url, ip="222.22.222.222")
url = url_for("dataset.read", id="use_of_weapons")
self._post_to_tracking(app, url, ip="111.11.111.111")
self._post_to_tracking(app, url, ip="222.22.222.222")
self._post_to_tracking(app, url, ip="333.33.333.333")
self._update_tracking_summary()
response = tests.call_action_api(
app, "package_search", sort="views_total desc"
)
assert response["count"] == 3
assert response["sort"] == "views_total desc"
packages = response["results"]
assert packages[0]["name"] == "use_of_weapons"
assert packages[1]["name"] == "the_player_of_games"
assert packages[2]["name"] == "consider_phlebas"
def test_package_with_many_views(self):
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
self._create_resource(app, package, apikey)
url = routes.url_for(controller='package', action='read',
id=package['name'])
# View the package three times from different IPs.
self._post_to_tracking(app, url, ip='111.222.333.44')
self._post_to_tracking(app, url, ip='111.222.333.55')
self._post_to_tracking(app, url, ip='111.222.333.66')
self._update_tracking_summary()
package = tests.call_action_api(app, 'package_show', id=package['id'],
include_tracking=True)
tracking_summary = package['tracking_summary']
assert tracking_summary['recent'] == 3, (
"A package that has been viewed 3 times recently should have 3 "
"recent views")
assert tracking_summary['total'] == 3, (
"A package that has been viewed 3 times should have 3 total views")
assert len(package['resources']) == 1
resource = package['resources'][0]
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 0, (
"Viewing a package should not increase the recent views of the "
"package's resources")
assert tracking_summary['total'] == 0, (
"Viewing a package should not increase the total views of the "
"package's resources")
def test_create_org(self):
user = model.User.by_name('tester')
tests.call_action_api(self.app,
'organization_create',
name='test_org_a',
title='Test Org A',
apikey=user.apikey,
status=403)
testsysadmin = model.User.by_name('testsysadmin')
tests.call_action_api(self.app,
'organization_create',
name='test_org_a_admin',
title='Test Org A Admin',
apikey=testsysadmin.apikey,
status=200)
assert True
def test_group_create_with_no_curators_group(self):
'''Test that group_create returns a 404 when there's no curators group.
With this version of the plugin group_create returns a spurious 404
when a user _is_ logged-in but the site has no curators group.
'''
assert 'curators' not in tests.call_action_api(self.app, 'group_list')
user = factories.User()
response = tests.call_action_api(self.app, 'group_create',
name='test_group',
apikey=user['apikey'], status=404)
assert response == {'__type': 'Not Found Error',
'message': 'Not found'}
def setup_class(cls):
CreateTestData.create()
cls.tester_user = model.User.by_name(u'tester')
cls.extra_environ_admin = {'REMOTE_USER': '******'}
cls.extra_environ_tester = {'REMOTE_USER': '******'}
cls.extra_environ_someone_else = {'REMOTE_USER': '******'}
tests.call_action_api(cls.app, 'organization_create',
name='test_org_2',
apikey=cls.tester_user.apikey)
tests.call_action_api(cls.app, 'package_create',
name='crimeandpunishment',
owner_org='test_org_2',
apikey=cls.tester_user.apikey)
def test_package_with_one_view(self):
app = self._get_app()
sysadmin_user, apikey = self._create_sysadmin(app)
package = self._create_package(app, apikey)
self._create_resource(app, package, apikey)
url = routes.url_for(controller='package',
action='read',
id=package['name'])
self._post_to_tracking(app, url)
self._update_tracking_summary()
package = tests.call_action_api(app,
'package_show',
id=package['id'],
include_tracking=True)
tracking_summary = package['tracking_summary']
assert tracking_summary['recent'] == 1, ("A package that has been "
"viewed once should have 1 "
"recent view.")
assert tracking_summary['total'] == 1, ("A package that has been "
"viewed once should have 1 "
"total view")
assert len(package['resources']) == 1
resource = package['resources'][0]
tracking_summary = resource['tracking_summary']
assert tracking_summary['recent'] == 0, ("Viewing a package should "
"not increase the recent "
"views of the package's "
"resources")
assert tracking_summary['total'] == 0, ("Viewing a package should "
"not increase the total views "
"of the package's resources")
def _create_package(self, app, apikey, name="look_to_windward"):
"""Create a package via the action api."""
return tests.call_action_api(app,
"package_create",
apikey=apikey,
name=name)
def test_00_email_notifications_disabled_by_default(self):
'''Email notifications should be disabled for new users.'''
# Register a new user.
sara = tests.call_action_api(self.app, 'user_create',
apikey=self.testsysadmin['apikey'], name='sara',
email='*****@*****.**', password='******',
fullname='Sara Rollins')
# Save the user for later tests to use.
TestEmailNotificationsUserPreference.sara = sara
# Email notifications should be disabled for the new user.
assert sara['activity_streams_email_notifications'] is False
assert (tests.call_action_api(self.app, 'user_show',
apikey=self.sara['apikey'], id='sara')[
'activity_streams_email_notifications'] is False)
def test_package_update_duplicate_extras_error(self):
import paste.fixture
import pylons.test
# We need to create a package first, so that we can update it.
app = helpers._get_test_app()
package = call_action_api(app, 'package_create',
apikey=self.sysadmin_user.apikey, name='foobar')
# Posting a dataset dict to package_update containing two extras dicts
# with the same key, should return a Validation Error.
package['extras'] = [{'key': 'foo', 'value': 'bar'},
{'key': 'foo', 'value': 'gar'}]
error = call_action_api(app, 'package_update',
apikey=self.sysadmin_user.apikey, status=409, **package)
assert error['__type'] == 'Validation Error'
assert error['extras_validation'] == ['Duplicate key "foo"']
def setup_class(cls):
if not tests.is_datastore_supported():
raise nose.SkipTest("Datastore not supported")
p.load('datastore')
ctd.CreateTestData.create()
cls.sysadmin_user = model.User.get('testsysadmin')
cls.normal_user = model.User.get('annafan')
cls.dataset = model.Package.get('annakarenina')
cls.resource = cls.dataset.resources[0]
cls.data = {
'resource_id': cls.resource.id,
'force': True,
'aliases': 'books3',
'fields': [{'id': u'b\xfck', 'type': 'text'},
{'id': 'author', 'type': 'text'},
{'id': 'published'},
{'id': u'characters', u'type': u'_text'},
{'id': 'rating with %'}],
'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}],
u'characters': [u'Princess Anna', u'Sergius'],
'rating with %': '60%'},
{u'b\xfck': 'warandpeace', 'author': 'tolstoy',
'nested': {'a': 'b'}, 'rating with %': '99%'}
]
}
postparams = '%s=1' % json.dumps(cls.data)
auth = {'Authorization': str(cls.sysadmin_user.apikey)}
res = cls.app.post('/api/action/datastore_create', params=postparams,
extra_environ=auth)
res_dict = json.loads(res.body)
assert res_dict['success'] is True
# Make an organization, because private datasets must belong to one.
cls.organization = tests.call_action_api(
cls.app, 'organization_create',
name='test_org',
apikey=cls.sysadmin_user.apikey)
cls.expected_records = [{u'published': u'2005-03-01T00:00:00',
u'_id': 1,
u'nested': [u'b', {u'moo': u'moo'}],
u'b\xfck': u'annakarenina',
u'author': u'tolstoy',
u'characters': [u'Princess Anna', u'Sergius'],
u'rating with %': u'60%'},
{u'published': None,
u'_id': 2,
u'nested': {u'a': u'b'},
u'b\xfck': u'warandpeace',
u'author': u'tolstoy',
u'characters': None,
u'rating with %': u'99%'}]
engine = db._get_engine(
{'connection_url': pylons.config['ckan.datastore.write_url']}
)
cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
def test_05_no_email_if_seen_on_dashboard(self):
'''Test that emails are not sent for activities already seen on dash.
If a user gets some new activities in her dashboard activity stream,
then views her dashboard activity stream, then she should not got any
email notifications about these new activities.
'''
# Make someone else update the dataset Sara's following, this should
# create a new activity on Sara's dashboard.
tests.call_action_api(self.app, 'package_update',
apikey=self.joeadmin['apikey'], name='warandpeace',
notes='updated by test_05_no_email_if_seen_on_dashboard')
# At this point Sara should have a new activity on her dashboard.
num_new_activities = tests.call_action_api(self.app,
'dashboard_new_activities_count', apikey=self.sara['apikey'])
assert num_new_activities > 0, num_new_activities
# View Sara's dashboard.
tests.call_action_api(self.app, 'dashboard_mark_activities_old',
apikey=self.sara['apikey'])
# No email should be sent.
tests.call_action_api(self.app, 'send_email_notifications',
apikey=self.testsysadmin['apikey'])
assert len(self.get_smtp_messages()) == 0
def test_create_private_package_with_no_organization(self):
'''Test that private packages with no organization cannot be created.
'''
testsysadmin = model.User.by_name('testsysadmin')
result = tests.call_action_api(self.app, 'package_create', name='test',
private=True, apikey=testsysadmin.apikey, status=409)
assert result == {'__type': 'Validation Error',
'private': ["Datasets with no organization can't be private."]}
def _call_datapusher_hook(self, user):
package = model.Package.get('annakarenina')
resource = package.resources[0]
context = {
'user': self.sysadmin_user.name
}
p.toolkit.get_action('task_status_update')(context, {
'entity_id': resource.id,
'entity_type': 'resource',
'task_type': 'datapusher',
'key': 'datapusher',
'value': '{"job_id": "my_id", "job_key":"my_key"}',
'last_updated': str(datetime.datetime.now()),
'state': 'pending'
})
data = {
'status': 'success',
'metadata': {
'resource_id': resource.id
}
}
postparams = '%s=1' % json.dumps(data)
auth = {'Authorization': str(user.apikey)}
res = self.app.post('/api/action/datapusher_hook', params=postparams,
extra_environ=auth, status=200)
print res.body
res_dict = json.loads(res.body)
assert res_dict['success'] is True
task = tests.call_action_api(
self.app, 'task_status_show', entity_id=resource.id,
task_type='datapusher', key='datapusher')
assert task['state'] == 'success', task
task = tests.call_action_api(
self.app, 'task_status_show', entity_id=resource.id,
task_type='datapusher', key='datapusher')
assert task['state'] == 'success', task
def test_group_create_with_curator(self):
'''A member of the curators group should be able to create a group.
'''
noncurator, curator, curators_group = self._make_curators_group()
name = 'my-new-group'
result = tests.call_action_api(self.app, 'group_create',
name=name,
apikey=curator['apikey'])
assert result['name'] == name
def test_pass_the_received_ignore_hash_param_to_the_datapusher(self):
pylons.config['datapusher.url'] = 'http://datapusher.ckan.org'
httpretty.HTTPretty.register_uri(
httpretty.HTTPretty.POST,
'http://datapusher.ckan.org/job',
content_type='application/json',
body=json.dumps({'job_id': 'foo', 'job_key': 'bar'}))
package = model.Package.get('annakarenina')
resource = package.resources[0]
tests.call_action_api(
self.app, 'datapusher_submit', apikey=self.sysadmin_user.apikey,
resource_id=resource.id,
ignore_hash=True
)
data = json.loads(httpretty.last_request().body)
assert data['metadata']['ignore_hash'], data
def setup_class(cls):
cls.app = paste.fixture.TestApp(pylons.test.pylonsapp)
# Make a sysadmin user.
cls.sysadmin = model.User(name='test_sysadmin', sysadmin=True)
model.Session.add(cls.sysadmin)
model.Session.commit()
model.Session.remove()
# A package that will be added to our test groups and organizations.
cls.package = tests.call_action_api(cls.app, 'package_create',
name='test_package',
apikey=cls.sysadmin.apikey)
# A user who will not be a member of our test groups or organizations.
cls.visitor = tests.call_action_api(cls.app, 'user_create',
name='non_member',
email='blah',
password='******',
apikey=cls.sysadmin.apikey)
# A user who will become a member of our test groups and organizations.
cls.member = tests.call_action_api(cls.app, 'user_create',
name='member',
email='blah',
password='******',
apikey=cls.sysadmin.apikey)
# A user who will become an editor of our test groups and
# organizations.
cls.editor = tests.call_action_api(cls.app, 'user_create',
name='editor',
email='blah',
password='******',
apikey=cls.sysadmin.apikey)
# A user who will become an admin of our test groups and organizations.
cls.admin = tests.call_action_api(cls.app, 'user_create',
name='admin',
email='blah',
password='******',
apikey=cls.sysadmin.apikey)
def _test_group_or_org_purge_with_missing_id(self, is_org):
if is_org:
action = 'organization_purge'
else:
action = 'group_purge'
result = tests.call_action_api(self.app, action,
apikey=self.sysadmin.apikey,
status=409,
)
assert result == {'__type': 'Validation Error',
'id': ['Missing value']}
def test_group_create_with_non_curator(self):
'''A user who isn't a member of the curators group should not be able
to create a group.
'''
noncurator, curator, curators_group = self._make_curators_group()
result = tests.call_action_api(self.app, 'group_create',
name='this_group_should_not_be_created',
apikey=noncurator['apikey'],
status=403)
assert result['__type'] == 'Authorization Error'
def _organization_create(self, organization_name):
'''Return an organization with some users and a dataset.'''
# Make an organization with some users.
users = [{'name': self.member['name'], 'capacity': 'member'},
{'name': self.editor['name'], 'capacity': 'editor'},
{'name': self.admin['name'], 'capacity': 'admin'}]
organization = tests.call_action_api(self.app, 'organization_create',
apikey=self.sysadmin.apikey,
name=organization_name,
users=users)
# Add a dataset to the organization (have to do this separately
# because the packages param of organization_create doesn't work).
tests.call_action_api(self.app, 'package_update',
name=self.package['name'],
owner_org=organization['name'],
apikey=self.sysadmin.apikey)
return organization
def test_group_create_with_visitor(self):
'''Test that group_create returns 403 when no one is logged in.
Since #1210 non-logged in requests are automatically rejected, unless
the auth function has the appropiate decorator
'''
noncurator, curator, curators_group = self._make_curators_group()
response = tests.call_action_api(self.app, 'group_create',
name='this_group_shouldnt_be_created',
status=403)
assert response['__type'] == 'Authorization Error'
def test_group_create_with_visitor(self):
'''A visitor (not logged in) should not be able to create a group.
Note: this also tests that the group_create auth function doesn't
crash when the user isn't logged in.
'''
noncurator, curator, curators_group = self._make_curators_group()
result = tests.call_action_api(self.app, 'group_create',
name='this_group_should_not_be_created',
status=403)
assert result['__type'] == 'Authorization Error'
