def setup(self, get_total_submissions, create_kobo_resources):
get_total_submissions.return_value = 5 # any, > 0
self.kobo_user = factories.InternalKoBoUser()
self.internal_user = factories.InternalUser()
self.data_container = factories.DataContainer(users=[
{
'name': self.kobo_user['name'],
'capacity': 'editor'
},
{
'name': self.internal_user['name'],
'capacity': 'member'
},
])
self.kobo_test_asset_id = 'test_kobo_id_1234'
self.kobo_dataset = factories.Dataset(
name='kobo-dataset',
title='KoBo Dataset',
owner_org=self.data_container['name'],
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.kobo_user,
kobo_asset_id=self.kobo_test_asset_id,
)
base_resource = dict(name='kobo-resource',
package_id='kobo-dataset',
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
url='original-file.csv',
format='csv',
kobo_type='data',
kobo_details={
'kobo_export_id':
'test_export_id',
'kobo_asset_id':
self.kobo_test_asset_id,
'kobo_download_status':
'pending',
'kobo_download_attempts':
1,
'kobo_submission_count':
3,
'kobo_last_updated':
datetime.datetime.utcnow().isoformat(),
})
self.kobo_resource = factories.Resource(**base_resource)
base_resource['kobo_details']['kobo_download_attempts'] = 5
base_resource['upload'] = mocks.FakeFileStorage()
self.kobo_resource_last_attempt = factories.Resource(**base_resource)
def test_edit_kobo_resource_must_preserve_upload(self, submissions,
create_kobo_resources,
app):
# required to create KoBo dataset
submissions.return_value = 1
self.kobo_dataset = factories.Dataset(
name='kobo-dataset',
title='KoBo Dataset',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.kobo_user,
kobo_asset_id='test_1234',
)
self.kobo_resource = factories.Resource(name='kobo-resource',
package_id='kobo-dataset',
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
url='original-file.csv',
kobo_type='data')
url = toolkit.url_for('resource.edit',
id=self.kobo_dataset['id'],
resource_id=self.kobo_resource['id'])
env = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
# Mock a resource edit payload
form_data = {
'description': 'updated',
'url_type': 'upload',
'upload': mocks.FakeFileStorage(filename='different-file.csv'),
# this is manually added to the update form as hidden fields:
'original_url': self.kobo_resource['url'],
'kobo_type': 'data',
'url': 'different-file.csv',
'clear_upload': '',
'save': ''
}
data = dict(self.kobo_resource, **form_data)
resp = app.post(url, data=data, extra_environ=env, status=200)
assert 'The form contains invalid entries:' in resp.body
assert 'You cannot update a KoBoToolbox data file directly, please re-import the data instead' in resp.body
def setup(self):
# Users
self.normal_user = core_factories.User()
self.org_user = core_factories.User()
self.sysadmin = core_factories.Sysadmin()
# Containers
self.container = factories.DataContainer(
users=[
{'name': self.org_user['name'], 'capacity': 'member'},
]
)
# Datasets
self.dataset = factories.Dataset(
owner_org=self.container['id'],
)
self.resource = factories.Resource(
package_id=self.dataset['id'],
url_type='datastore',
upload=mocks.FakeFileStorage(),
visibility='restricted',
)
# Actions
core_helpers.call_action('datastore_create',
resource_id=self.resource['id'],
records=[{'a': 1, 'b': 2}],
force=True,
)
def test_upload_present_after_update(self):
dataset = factories.Dataset()
resource = factories.Resource(
package_id=dataset['id'],
upload=mocks.FakeFileStorage(),
url = "http://fakeurl/test.txt",
url_type='upload',
)
resource['name'] = 'updated'
updated_resource = core_helpers.call_action(
'resource_update',
{'ignore_auth': True},
**resource
)
assert updated_resource['name'] == 'updated'
assert (
updated_resource['url'] ==
'{}/dataset/{}/resource/{}/download/test.txt'.format(
toolkit.config['ckan.site_url'].rstrip('/'),
dataset['id'],
resource['id']
)
)
def setup(self):
# Users
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.user1 = core_factories.User(name='user1', id='user1')
self.user2 = core_factories.User(name='user2', id='user2')
self.user3 = core_factories.User(name='user3', id='user3')
# Containers
self.container1 = factories.DataContainer(
name='container1',
id='container1',
users=[
{
'name': 'user1',
'capacity': 'admin'
},
],
)
self.container2 = factories.DataContainer(
name='container2',
id='container2',
users=[
{
'name': 'user2',
'capacity': 'admin'
},
],
)
# Datasets
self.dataset1 = factories.Dataset(
name='dataset1',
title='Test Dataset 1',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.user1,
tags=[{
'name': 'Keyword1'
}, {
'name': 'Keyword2'
}],
)
# Resources
self.resource1 = factories.Resource(
name='resource1',
package_id='dataset1',
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
def setup(self):
self.internal_user = core_factories.User()
self.dataset = factories.Dataset(
user=self.internal_user
)
self.resource = factories.Resource(
package_id=self.dataset['id'],
description='some description',
visibility='public',
identifiability='personally_identifiable',
upload=mocks.FakeFileStorage(),
url="http://fakeurl/test.txt",
url_type='upload',
)
self.resource_visible = factories.Resource(
package_id=self.dataset['id'],
description='some description 2',
visibility='public',
identifiability='anonymized_public',
upload=mocks.FakeFileStorage(),
url="http://fakeurl/test2.txt",
url_type='upload',
)
# Attachment type files do not use "identifiability"
self.resource_ambiguous = factories.Resource(
package_id=self.dataset['id'],
description='some description 3',
visibility='public',
identifiability='personally_identifiable',
upload=mocks.FakeFileStorage(),
url="http://fakeurl/test3.txt",
url_type='upload',
type='attachment',
file_type='other',
)
def test_visibility_private(self, app):
dataset = factories.Dataset(
private=True,
visibility='private',
owner_org=self.container['id'],
)
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
# Even though we tried to save this with visibility='private'
# this feature is disabled, so the validator has altered it for us
# from now on the dataset should behave as 'restricted'
assert dataset['visibility'] == 'restricted'
resource_download_url = toolkit.url_for('resource.download',
id=dataset['id'],
resource_id=resource['id'])
dataset_read_url = toolkit.url_for('dataset.read', id=dataset['id'])
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
in res.body)
app.get(resource_download_url, extra_environ=environ, status=403)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
def test_upload_present(self):
dataset = factories.Dataset()
resource = factories.Resource(
package_id=dataset['id'],
upload=mocks.FakeFileStorage(),
url = "http://fakeurl/test.txt",
url_type='upload',
)
assert (
resource['url'] ==
'{}/dataset/{}/resource/{}/download/test.txt'.format(
toolkit.config.get('ckan.site_url').rstrip('/'),
dataset['id'],
resource['id']
)
)
def test_both_types_data_fields_missing(self):
""" Test we have one error in the second spot of the error list
Requires 2 resources. Only one invalid. """
dataset = factories.Dataset()
resource1 = {
'name': 'Test File attachment',
'url': 'http://example.com/doc.pdf',
'format': 'PDF',
'description': 'Some description',
'type': 'attachment',
'file_type': 'report',
'visibility': 'public'
}
resource2 = {
'name': 'Test Data file',
'upload': mocks.FakeFileStorage(),
'format': 'CSV',
'description': 'Some data file',
'type': 'data',
'file_type': 'microdata',
}
dataset['resources'] = [resource1, resource2]
with pytest.raises(toolkit.ValidationError) as e:
call_action('package_update', {'user': self.sysadmin['name']},
**dataset)
# First resource it's ok
assert e.value.error_dict['resources'][0] == {}
errors = e.value.error_dict['resources'][1]
# TODO add 'visibility' after merge #583
for field in [
'identifiability', 'date_range_end', 'version',
'date_range_start', 'process_status'
]:
error = errors[field]
assert error == ['Missing value']
def setup(self):
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.container1_admin = core_factories.User()
self.container1_member = core_factories.User()
self.container1 = factories.DataContainer(users=[
{
"name": self.container1_admin["name"],
"capacity": "admin"
},
{
"name": self.container1_member["name"],
"capacity": "member"
},
])
self.dataset1 = factories.Dataset(owner_org=self.container1["id"])
self.resource1 = factories.Resource(
package_id=self.dataset1['id'],
upload=mocks.FakeFileStorage(),
visibility="restricted",
url="http://fakeurl/test.txt",
url_type='upload',
)
create_curation_activity('dataset_approved',
self.dataset1['id'],
self.dataset1['name'],
self.sysadmin['id'],
message='asdf')
create_download_activity({'user': self.sysadmin['name']},
self.resource1['id'])
toolkit.get_action('activity_create')({
'ignore_auth': True
}, {
'user_id': self.sysadmin['name'],
'object_id': self.dataset1['id'],
'activity_type': 'changed package',
'data': {}
})
def test_visibility_restricted(self, app):
dataset = factories.Dataset(owner_org=self.container['id'], )
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
resource_download_url = toolkit.url_for('resource.download',
id=dataset['id'],
resource_id=resource['id'])
dataset_read_url = toolkit.url_for('dataset.read', id=dataset['id'])
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
in res.body)
app.get(resource_download_url, extra_environ=environ, status=403)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
def test_publish_dataset_with_resources(self, app):
draft_dataset = factories.Dataset(
name='dataset2',
title='Test Dataset 2',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.user1,
state='draft',
)
factories.Resource(
name='resource1',
package_id=draft_dataset['name'],
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
self.make_dataset_publish_request(
app,
draft_dataset['id'],
user='******',
status=200,
)