def test_get_linked_datasets_for_form_many_selected_ids(self):
user = core_factories.User(id='user_selected_ids',
name='user_selected_ids')
container1 = factories.DataContainer(title='container1', users=[user])
container2 = factories.DataContainer(title='container2', users=[user])
dataset1 = factories.Dataset(id='id1',
title='dataset1',
owner_org=container1['id'])
dataset2 = factories.Dataset(id='id2',
title='dataset2',
owner_org=container2['id'])
context = {'model': model, 'user': user['name']}
linked_datasets = get_linked_datasets_for_form(context=context,
user_id=user['id'],
selected_ids=['id2'])
assert_equals(linked_datasets, [
{
'text': 'container1',
'children': [{
'text': 'dataset1',
'value': 'id1'
}]
},
{
'text':
'container2',
'children': [{
'text': 'dataset2',
'value': 'id2',
'selected': 'selected'
}]
},
])
def setup(self):
# these geographies are all related in a partent/child hierarchy
self.hierarchy = {
'IRQ': factories.Geography(
pcode='IRQ',
iso3='IRQ',
gis_name='Iraq',
hierarchy_pcode='IRQ'
),
'20IRQ015': factories.Geography(
pcode='20IRQ015',
iso3='IRQ',
gis_name='Ninewa',
hierarchy_pcode='20IRQ015',
),
'20IRQ015004': factories.Geography(
pcode='20IRQ015004',
iso3='IRQ',
gis_name='Mosul',
hierarchy_pcode='20IRQ015004',
),
'20IRQ015004159': factories.Geography(
pcode='IRQr000019225',
iso3='IRQ',
gis_name='Mosul',
hierarchy_pcode='20IRQ015004159',
),
}
# some other geographies that aren't related to our 'main' hierarchy
self.unrelated = {
'BRZ': factories.Geography(
pcode='BRZ', iso3='BRZ', gis_name='Brazil', hierarchy_pcode='BRZ'
),
'20DEU010004': factories.Geography(
pcode='20DEU010004',
iso3='DEU',
gis_name='Regierungsbezirk Dusseldorf',
hierarchy_pcode='20DEU010004',
),
}
self.geogs = {**self.hierarchy, **self.unrelated}
self.no_gis_dataset = factories.Dataset()
self.gis_dataset1 = factories.Dataset(
name="gis1",
geographies=','.join([self.hierarchy['20IRQ015004159'].pcode, self.unrelated['BRZ'].pcode])
)
self.gis_dataset2 = factories.Dataset(
name="gis2",
geographies=self.unrelated['20DEU010004'].pcode
)
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
def setup(self):
# Users
self.normal_user = core_factories.User()
self.org_user = core_factories.User()
self.sysadmin = core_factories.Sysadmin()
# Containers
self.container = factories.DataContainer(
users=[
{'name': self.org_user['name'], 'capacity': 'member'},
]
)
# Datasets
self.dataset = factories.Dataset(
owner_org=self.container['id'],
)
self.resource = factories.Resource(
package_id=self.dataset['id'],
url_type='datastore',
upload=mocks.FakeFileStorage(),
visibility='restricted',
)
# Actions
core_helpers.call_action('datastore_create',
resource_id=self.resource['id'],
records=[{'a': 1, 'b': 2}],
force=True,
)
def setup(self):
super(TestDatastoreAuthRestrictedDownloads, self).setup()
# Users
self.normal_user = core_factories.User()
self.org_user = core_factories.User()
self.sysadmin = core_factories.Sysadmin()
# Containers
self.container = factories.DataContainer(users=[
{
'name': self.org_user['name'],
'capacity': 'member'
},
])
# Datasets
self.dataset = factories.Dataset(
visibility='restricted',
owner_org=self.container['id'],
)
self.resource = factories.Resource(
package_id=self.dataset['id'],
url_type='datastore',
)
# Actions
core_helpers.call_action('datastore_create',
resource_id=self.resource['id'],
records=[{
'a': 1,
'b': 2
}])
def test_resource_download(self):
container_member = core_factories.User()
dataset_member = core_factories.User()
another_user = core_factories.User()
data_container = factories.DataContainer(
users=[{
'name': container_member['name'],
'capacity': 'admin'
}])
dataset = factories.Dataset(owner_org=data_container['id'], )
resource = factories.Resource(package_id=dataset['id'],
url_type='upload',
visibility='restricted')
helpers.call_action(
'package_collaborator_create',
id=dataset['id'],
user_id=dataset_member['id'],
capacity='member',
)
for user in [container_member, dataset_member]:
assert ({
'success': True
} == auth.resource_download({'user': user['name']}, resource))
assert ({
'success': False
} == auth.resource_download({'user': another_user['name']}, resource))
def test_access_visibility_private_pages_not_visible(self):
dataset = factories.Dataset(
private=True,
visibility='private',
owner_org=self.container['id'],
user=self.sysadmin,
)
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
)
url = toolkit.url_for('dataset_read', id=dataset['id'])
app = self._get_test_app()
environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')}
res = app.get(url, extra_environ=environ, status=404)
environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')}
res = app.get(url, extra_environ=environ)
assert_equals(res.status_int, 200)
environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
res = app.get(url, extra_environ=environ)
assert_equals(res.status_int, 200)
def test_private_can_be_true_if_sysadmin(self):
dataset = factories.Dataset(private=True,
visibility='private',
user=self.sysadmin)
assert_equals(dataset['private'], True)
def test_both_types_data_fields_missing(self):
dataset = factories.Dataset()
resource1 = {
'name': 'Test File attachment',
'url': 'http://example.com/doc.pdf',
'format': 'PDF',
'description': 'Some description',
'type': 'attachment',
'file_type': 'report',
}
resource2 = {
'name': 'Test Data file',
'url': 'http://example.com/data.csv',
'format': 'CSV',
'description': 'Some data file',
'type': 'data',
'file_type': 'microdata',
}
dataset['resources'] = [resource1, resource2]
with assert_raises(ValidationError) as e:
call_action('package_update', {}, **dataset)
errors = e.exception.error_dict['resources'][1]
for field in [
'identifiability', 'date_range_end', 'version',
'date_range_start', 'process_status'
]:
error = errors[field]
assert_equals(error, ['Missing value'])
def test_logged_in_users(self):
user = core_factories.User()
actions = [
'package_search',
'package_list',
'organization_list',
'group_list',
'user_list',
'organization_list_for_user',
]
context = {'user': user['name'], 'ignore_auth': False}
for action in actions:
helpers.call_action(action, context=context)
data_container = factories.DataContainer(users=[{
'name': user['name'],
'capacity': 'admin'
}])
dataset = factories.Dataset(owner_org=data_container['id'])
helpers.call_action('package_show',
context=context,
id=dataset['name'])
helpers.call_action('organization_show',
context=context,
id=data_container['name'])
helpers.call_action('user_show', context=context, id=user['id'])
def test_recipients_with_org_admins(self):
editor = core_factories.User()
admin = core_factories.User()
external = core_factories.User()
container = factories.DataContainer(users=[
{
'name': editor['name'],
'capacity': 'editor'
},
{
'name': admin['name'],
'capacity': 'admin'
},
],
name='container1',
id='container1')
dataset1 = factories.Dataset(
name='new-dataset',
title='New Dataset',
owner_org=container['id'],
)
dataset_recipients = mailer.get_dataset_request_access_email_recipients(
dataset1)
container_recipients = mailer.get_container_request_access_email_recipients(
container)
assert dataset_recipients == container_recipients
assert len(container_recipients) == 1
assert admin['name'] == container_recipients[0]['name']
def test_upload_present_after_update(self):
dataset = factories.Dataset()
resource = factories.Resource(
package_id=dataset['id'],
upload=mocks.FakeFileStorage(),
url = "http://fakeurl/test.txt",
url_type='upload',
)
resource['name'] = 'updated'
updated_resource = core_helpers.call_action(
'resource_update',
{'ignore_auth': True},
**resource
)
assert updated_resource['name'] == 'updated'
assert (
updated_resource['url'] ==
'{}/dataset/{}/resource/{}/download/test.txt'.format(
toolkit.config['ckan.site_url'].rstrip('/'),
dataset['id'],
resource['id']
)
)
def test_resource_is_blocked_task_status_infected(self):
user = core_factories.User()
dataset = factories.Dataset()
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
)
toolkit.get_action('task_status_update')(
{
'ignore_auth': True,
# task_status_update wants a user object
# for no reason, even with 'ignore_auth': True
# give it an empty string to keep it happy
'user': ''
},
{
'entity_id': resource['id'],
'entity_type': 'resource',
'task_type': 'clamav',
'last_updated': datetime.datetime.utcnow().isoformat(),
'state': 'complete',
'key': 'clamav',
'value': '{"data": {"status_code": 1}}',
'error': 'null',
}
)
assert utils.resource_is_blocked(
{'user': user['name']},
resource['id']
)
def test_file_attachment_fields(self):
dataset = factories.Dataset()
resource = {
'name': 'Test File attachment',
'url': 'http://example.com/doc.pdf',
'format': 'PDF',
'description': 'Some description',
'type': 'attachment',
'file_type': 'report',
'visibility': 'public',
'url_type': 'upload',
}
dataset['resources'] = [resource]
updated_dataset = call_action('package_update',
{'user': self.sysadmin['name']},
**dataset)
for field in [k for k in resource.keys() if k != 'url']:
assert updated_dataset['resources'][0][field] == resource[field]
assert (updated_dataset['resources'][0]['url'].split('/')[-1] ==
resource['url'].split('/')[-1])
assert 'date_range_start' not in updated_dataset['resources'][0]
def test_request_renewal(self, mock_notify_renewal_request):
user = factories.ExternalUser()
dataset = factories.Dataset()
activity = core_factories.Activity(
user_id=user["id"],
object_id=dataset["id"],
activity_type="new package",
data={
"package": dataset,
"actor": "Mr Someone"
},
)
mock_notify_renewal_request.return_value = []
created, reason = request_renewal(user, activity)
assert created
assert reason is None
mock_notify_renewal_request.assert_called_once()
call = mock_notify_renewal_request.call_args_list[0]
args, kwargs = call
user_id = kwargs['user_id']
message = kwargs['message']
assert user_id == user['id']
assert 'User {} will expire on'.format(user['name']) in message
assert 'Last activity: "{}" registered on'.format(
activity['activity_type']) in message
created, reason = request_renewal(user, activity)
assert not created
assert reason == 'The request already exists'
def test_access_visibility_private(self):
dataset = factories.Dataset(
visibility='private',
owner_org=self.container['id'],
)
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
)
url = toolkit.url_for(controller='package',
action='resource_download',
id=dataset['id'],
resource_id=resource['id'])
app = self._get_test_app()
# We don't pass authorization (forbidden)
environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')}
res = app.get(url, extra_environ=environ, status=403)
# We don't have data but we pass authorization
environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')}
res = app.get(url, extra_environ=environ, status=404)
# We don't have data but we pass authorization
environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
res = app.get(url, extra_environ=environ, status=404)
def test_data_file_fields(self):
dataset = factories.Dataset()
resource = {
'name': 'Test Data file',
'url': 'http://example.com/data.csv',
'format': 'CSV',
'description': 'Some data file',
'type': 'data',
'file_type': 'microdata',
}
dataset['resources'] = [resource]
with pytest.raises(toolkit.ValidationError) as e:
call_action('package_update', {'user': self.sysadmin['name']},
**dataset)
errors = e.value.error_dict['resources'][0]
for field in [
'identifiability', 'date_range_end', 'version',
'date_range_start', 'process_status'
]:
error = errors[field]
assert error == ['Missing value']
def setup(self):
self.sysadmin = core_factories.Sysadmin()
self.dataset = factories.Dataset()
self.resource = factories.Resource(
package_id=self.dataset['id'],
url_type='upload',
last_modified=datetime.datetime.utcnow(),
)
def test_convert_dataset_to_microdata_survey_minimal(self):
dataset = factories.Dataset(
operational_purpose_of_data='cartography',
name='dataset',
unit_of_measurement='individual',
keywords=['3', '4'],
archived='False',
data_collector=['acf'],
data_collection_technique='f2f',
sampling_procedure='nonprobability',
)
survey = helpers.convert_dataset_to_microdata_survey(dataset,
nation='nation',
repoid='repoid')
assert_equals(
survey, {
'repositoryid': 'REPOID',
'access_policy': 'na',
'published': 0,
'overwrite': 'no',
'study_desc': {
'title_statement': {
'idno': u'DATASET',
'title': u'Test Dataset'
},
'authoring_entity':
[{
'affiliation': 'UNHCR',
'name': 'Office of the High Commissioner for Refugees'
}],
'study_info': {
'topics': [{
'topic': 'Health'
}, {
'topic': 'Water Sanitation Hygiene'
}],
'nation': [{
'name': 'nation'
}],
'abstract':
'Just another test dataset.',
'analysis_unit':
'individual',
},
'method': {
'data_collection': {
'data_collectors': [
{
'name': 'Action contre la faim'
},
],
'sampling_procedure': 'Non-probability',
'coll_mode': 'Face-to-face interview',
},
},
},
})
def test_all_actions_bad_dataset_type(self):
factories.Dataset(name='regular',
owner_org='data-target',
user=self.creator)
for action in ACTIONS:
self.make_request(action,
dataset_id='regular',
user='******',
status=403)
def test_get_linked_datasets_for_display_many(self):
url = os.environ.get('CKAN_SITE_URL', 'http://test.ckan.net')
user = core_factories.User()
dataset1 = factories.Dataset(name='name1', title='title1')
dataset2 = factories.Dataset(name='name2', title='title2')
context = {'model': model, 'user': user['name']}
linked_datasets = get_linked_datasets_for_display(
'{%s,%s}' % (dataset1['id'], dataset2['id']), context=context)
assert_equals(linked_datasets, [
{
'href': '%s/dataset/name1' % url,
'text': 'title1'
},
{
'href': '%s/dataset/name2' % url,
'text': 'title2'
},
])
def test_email_notification_delete(self, mock_mail_user):
dataset = factories.Dataset()
user = core_factories.User()
capacity = 'editor'
mailer.mail_notification_to_collaborator(dataset['id'], user['id'],
capacity, 'delete')
assert mock_mail_user.call_count == 1
def setup(self):
# Config
toolkit.config['ckanext.unhcr.microdata_api_key'] = 'API-KEY'
# Users
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.user = core_factories.User(name='user', id='user')
# Datasets
self.dataset = factories.Dataset(name='dataset')
def setup(self, get_total_submissions, create_kobo_resources):
get_total_submissions.return_value = 5 # any, > 0
self.kobo_user = factories.InternalKoBoUser()
self.internal_user = factories.InternalUser()
self.data_container = factories.DataContainer(users=[
{
'name': self.kobo_user['name'],
'capacity': 'editor'
},
{
'name': self.internal_user['name'],
'capacity': 'member'
},
])
self.kobo_test_asset_id = 'test_kobo_id_1234'
self.kobo_dataset = factories.Dataset(
name='kobo-dataset',
title='KoBo Dataset',
owner_org=self.data_container['name'],
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.kobo_user,
kobo_asset_id=self.kobo_test_asset_id,
)
base_resource = dict(name='kobo-resource',
package_id='kobo-dataset',
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
url='original-file.csv',
format='csv',
kobo_type='data',
kobo_details={
'kobo_export_id':
'test_export_id',
'kobo_asset_id':
self.kobo_test_asset_id,
'kobo_download_status':
'pending',
'kobo_download_attempts':
1,
'kobo_submission_count':
3,
'kobo_last_updated':
datetime.datetime.utcnow().isoformat(),
})
self.kobo_resource = factories.Resource(**base_resource)
base_resource['kobo_details']['kobo_download_attempts'] = 5
base_resource['upload'] = mocks.FakeFileStorage()
self.kobo_resource_last_attempt = factories.Resource(**base_resource)
def test_request_rejected_email_body_dataset(self):
user1 = core_factories.User(name='user1', id='user1')
dataset1 = factories.Dataset(title='Test Dataset 1')
message = 'Nope\nNot today thank you'
email_body = mailer.compose_request_rejected_email_body(
'dataset', user1, dataset1, message)
regularised_body = regularise_html(email_body)
assert 'Your request to access <strong>Test Dataset 1</strong> has been rejected.' in regularised_body
assert '<p>Nope<br> Not today thank you</p>' in regularised_body
def setup(self):
# Users
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.user1 = core_factories.User(name='user1', id='user1')
self.user2 = core_factories.User(name='user2', id='user2')
self.user3 = core_factories.User(name='user3', id='user3')
# Containers
self.container1 = factories.DataContainer(
name='container1',
id='container1',
users=[
{
'name': 'user1',
'capacity': 'admin'
},
],
)
self.container2 = factories.DataContainer(
name='container2',
id='container2',
users=[
{
'name': 'user2',
'capacity': 'admin'
},
],
)
# Datasets
self.dataset1 = factories.Dataset(
name='dataset1',
title='Test Dataset 1',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.user1,
tags=[{
'name': 'Keyword1'
}, {
'name': 'Keyword2'
}],
)
# Resources
self.resource1 = factories.Resource(
name='resource1',
package_id='dataset1',
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
def test_resource_is_blocked_no_task_status(self):
user = core_factories.User()
dataset = factories.Dataset()
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
)
assert not utils.resource_is_blocked(
{'user': user['name']},
resource['id']
)
def test_external_user(self):
sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
external_user = factories.ExternalUser()
dataset = factories.Dataset()
action = toolkit.get_action("package_collaborator_create")
with pytest.raises(toolkit.ValidationError):
action({'user': sysadmin['name']}, {
'id': dataset['id'],
'user_id': external_user['id'],
'capacity': 'member',
})
def setup(self):
super(TestActions, self).setup()
# Config
config['ckanext.unhcr.microdata_api_key'] = 'API-KEY'
# Users
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.user = core_factories.User(name='user', id='user')
# Datasets
self.dataset = factories.Dataset(name='dataset')
def test_upload_missing(self):
dataset = factories.Dataset()
with assert_raises(toolkit.ValidationError) as exc:
factories.Resource(package_id=dataset['id'])
assert exc.exception.error_dict.keys() == ['upload']
assert_equals(exc.exception.error_dict['upload'],
['All resources require an uploaded file'])
def test_get_linked_datasets_for_display_one(self):
url = os.environ.get('CKAN_SITE_URL', 'http://test.ckan.net')
user = core_factories.User()
dataset = factories.Dataset(name='name', title='title')
context = {'model': model, 'user': user['name']}
linked_datasets = get_linked_datasets_for_display(dataset['id'],
context=context)
assert_equals(linked_datasets, [
{
'href': '%s/dataset/name' % url,
'text': 'title'
},
])