def test_upload_external_url_attachment(self):
dataset = factories.Dataset()
resource = factories.Resource(
type='attachment',
package_id=dataset['id'],
url='https://example.com/some.data.csv',
file_type='other'
)
# tbh, the main thing we're testing here is that the line above
# runs without throwing a ValidationError
# but I suppose we should assert _something_
assert resource['url'] == 'https://example.com/some.data.csv'
def test_resource_download_deposited_dataset(self):
depadmin = core_factories.User()
curator = core_factories.User()
target_container_admin = core_factories.User()
target_container_member = core_factories.User()
other_container_admin = core_factories.User()
deposit = factories.DataContainer(id='data-deposit',
users=[
{
'name': depadmin['name'],
'capacity': 'admin'
},
{
'name': curator['name'],
'capacity': 'editor'
},
])
target = factories.DataContainer(users=[
{
'name': target_container_admin['name'],
'capacity': 'admin'
},
{
'name': target_container_member['name'],
'capacity': 'member'
},
])
container = factories.DataContainer(users=[
{
'name': other_container_admin['name'],
'capacity': 'admin'
},
])
dataset = factories.DepositedDataset(owner_org=deposit['id'],
owner_org_dest=target['id'])
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
)
for user in [depadmin, curator, target_container_admin]:
assert ({
'success': True
} == auth.resource_download({'user': user['name']}, resource))
for user in [target_container_member, other_container_admin]:
assert ({
'success': False
} == auth.resource_download({'user': user['name']}, resource))
def setup(self):
# Users
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.user1 = core_factories.User(name='user1', id='user1')
self.user2 = core_factories.User(name='user2', id='user2')
self.user3 = core_factories.User(name='user3', id='user3')
self.kobo_user = factories.InternalKoBoUser(name='kobo_user',
id='kobo_user')
# Containers
self.container1 = factories.DataContainer(
name='container1',
id='container1',
users=[
{
'name': 'user1',
'capacity': 'admin'
},
],
)
self.container2 = factories.DataContainer(
name='container2',
id='container2',
users=[
{
'name': 'user2',
'capacity': 'admin'
},
],
)
# Datasets
self.dataset1 = factories.Dataset(
name='dataset1',
title='Test Dataset 1',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.user1,
)
# Resources
self.resource1 = factories.Resource(
name='resource1',
package_id='dataset1',
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
def setup(self):
self.internal_user = core_factories.User()
self.dataset = factories.Dataset(
user=self.internal_user
)
self.resource = factories.Resource(
package_id=self.dataset['id'],
description='some description',
visibility='public',
identifiability='personally_identifiable',
upload=mocks.FakeFileStorage(),
url="http://fakeurl/test.txt",
url_type='upload',
)
self.resource_visible = factories.Resource(
package_id=self.dataset['id'],
description='some description 2',
visibility='public',
identifiability='anonymized_public',
upload=mocks.FakeFileStorage(),
url="http://fakeurl/test2.txt",
url_type='upload',
)
# Attachment type files do not use "identifiability"
self.resource_ambiguous = factories.Resource(
package_id=self.dataset['id'],
description='some description 3',
visibility='public',
identifiability='personally_identifiable',
upload=mocks.FakeFileStorage(),
url="http://fakeurl/test3.txt",
url_type='upload',
type='attachment',
file_type='other',
)
def test_edit_kobo_resource_must_preserve_upload(self, submissions,
create_kobo_resources,
app):
# required to create KoBo dataset
submissions.return_value = 1
self.kobo_dataset = factories.Dataset(
name='kobo-dataset',
title='KoBo Dataset',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.kobo_user,
kobo_asset_id='test_1234',
)
self.kobo_resource = factories.Resource(name='kobo-resource',
package_id='kobo-dataset',
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
url='original-file.csv',
kobo_type='data')
url = toolkit.url_for('resource.edit',
id=self.kobo_dataset['id'],
resource_id=self.kobo_resource['id'])
env = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
# Mock a resource edit payload
form_data = {
'description': 'updated',
'url_type': 'upload',
'upload': mocks.FakeFileStorage(filename='different-file.csv'),
# this is manually added to the update form as hidden fields:
'original_url': self.kobo_resource['url'],
'kobo_type': 'data',
'url': 'different-file.csv',
'clear_upload': '',
'save': ''
}
data = dict(self.kobo_resource, **form_data)
resp = app.post(url, data=data, extra_environ=env, status=200)
assert 'The form contains invalid entries:' in resp.body
assert 'You cannot update a KoBoToolbox data file directly, please re-import the data instead' in resp.body
def test_upload_external_url_data(self):
dataset = factories.Dataset()
with pytest.raises(toolkit.ValidationError) as exc:
factories.Resource(
type='data',
package_id=dataset['id'],
url='https://example.com/some.data.csv'
)
assert list(exc.value.error_dict.keys()) == ['url']
assert (
exc.value.error_dict['url'] ==
['All data resources require an uploaded file']
)
def test_upload_present(self):
dataset = factories.Dataset()
test_file = StringIO()
test_file.write('Some data')
test_upload = FakeFileStorage(test_file, "test.txt")
resource = factories.Resource(package_id=dataset['id'],
upload=test_upload)
assert_equals(
resource['url'],
'{}/dataset/{}/resource/{}/download/test.txt'.format(
config['ckan.site_url'].rstrip('/'), dataset['id'],
resource['id']))
def setup(self):
self.user = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.kobo_user = factories.InternalKoBoUser(name='kobo_user',
id='kobo_user')
self.container = factories.DataContainer(users=[
{
'name': self.kobo_user['name'],
'capacity': 'editor'
},
])
self.dataset = factories.Dataset(owner_org=self.container['id'])
self.resource = factories.Resource(
package_id=self.dataset['id'],
url_type='upload',
)
self.new_package_dict = {
'external_access_level': 'public_use',
'keywords': ['1'],
'archived': 'False',
'data_collector': 'test',
'data_collection_technique': 'nf',
'name': 'test',
'notes': 'test',
'unit_of_measurement': 'test',
'title': 'test',
'owner_org': self.container['id'],
'state': 'active',
'geographies': DEFAULT_GEOGRAPHY_CODE,
}
self.new_resource_dict = {
'package_id': self.dataset['id'],
'url': 'http://fakeurl/test.txt',
'url_type': 'upload',
'type': 'data',
'file_type': 'microdata',
'identifiability': 'anonymized_public',
'date_range_start': '2018-01-01',
'date_range_end': '2019-01-01',
'process_status': 'anonymized',
'visibility': 'public',
'version': '1',
}
def test_logged_in_users_private_dataset(self, app):
container_member = core_factories.User()
dataset_member = core_factories.User()
other_user = core_factories.User()
data_container = factories.DataContainer(
users=[{
'name': container_member['name'],
'capacity': 'admin'
}])
dataset = factories.Dataset(owner_org=data_container['id'], )
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
visibility='restricted',
)
helpers.call_action(
'package_collaborator_create',
id=dataset['id'],
user_id=dataset_member['id'],
capacity='member',
)
for user in [container_member, dataset_member]:
with app.flask_app.test_request_context():
response = app.get(
'/dataset/{}'.format(dataset['name']),
extra_environ={'REMOTE_USER': str(user['name'])},
status=200,
)
# these users can see the dataset_read view
assert 'You must be logged in' not in response.body
# these users can also download the resource attached to dataset
assert 'You are not authorized to download the private resources from this dataset' not in response.body
response = app.get(
'/dataset/{}'.format(dataset['name']),
extra_environ={'REMOTE_USER': str(other_user['name'])},
status=200,
)
# other_user is allowed to see the dataset_read view too
assert 'You must be logged in' not in response.body
# other_user is not allowed to download the resource
assert 'You are not authorized to download the private resources from this dataset' in response.body
# but they can request access to it if they like
assert 'If you need access to the dataset' in response.body
def test_visibility_private(self, app):
dataset = factories.Dataset(
private=True,
visibility='private',
owner_org=self.container['id'],
)
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
# Even though we tried to save this with visibility='private'
# this feature is disabled, so the validator has altered it for us
# from now on the dataset should behave as 'restricted'
assert dataset['visibility'] == 'restricted'
resource_download_url = toolkit.url_for('resource.download',
id=dataset['id'],
resource_id=resource['id'])
dataset_read_url = toolkit.url_for('dataset.read', id=dataset['id'])
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
in res.body)
app.get(resource_download_url, extra_environ=environ, status=403)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
def test_access_visibility_restricted(self):
dataset = factories.Dataset(visibility='restricted')
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
)
url = toolkit.url_for(controller='package',
action='resource_download',
id=dataset['id'],
resource_id=resource['id'])
app = self._get_test_app()
# We don't pass authorization (forbidden)
environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')}
res = app.get(url, extra_environ=environ, status=403)
def setup(self):
super(TestExtendedPackageController, self).setup()
# Users
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.user1 = core_factories.User(name='user1', id='user1')
self.user2 = core_factories.User(name='user2', id='user2')
self.user3 = core_factories.User(name='user3', id='user3')
# Containers
self.container1 = factories.DataContainer(
name='container1',
id='container1',
users=[
{
'name': 'user1',
'capacity': 'admin'
},
],
)
self.container2 = factories.DataContainer(
name='container2',
id='container2',
users=[
{
'name': 'user2',
'capacity': 'admin'
},
],
)
# Datasets
self.dataset1 = factories.Dataset(
name='dataset1',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.user1)
# Resources
self.resource1 = factories.Resource(name='resource1',
package_id='dataset1')
def test_upload_present(self):
dataset = factories.Dataset()
resource = factories.Resource(
package_id=dataset['id'],
upload=mocks.FakeFileStorage(),
url = "http://fakeurl/test.txt",
url_type='upload',
)
assert (
resource['url'] ==
'{}/dataset/{}/resource/{}/download/test.txt'.format(
toolkit.config.get('ckan.site_url').rstrip('/'),
dataset['id'],
resource['id']
)
)
def test_visibility_restricted(self, app):
dataset = factories.Dataset(owner_org=self.container['id'], )
resource = factories.Resource(
package_id=dataset['id'],
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
resource_download_url = toolkit.url_for('resource.download',
id=dataset['id'],
resource_id=resource['id'])
dataset_read_url = toolkit.url_for('dataset.read', id=dataset['id'])
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
in res.body)
app.get(resource_download_url, extra_environ=environ, status=403)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
with app.flask_app.test_request_context():
environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')}
res = app.get(dataset_read_url, extra_environ=environ, status=200)
assert (
'You are not authorized to download the private resources from this dataset'
not in res.body)
app.get(resource_download_url, extra_environ=environ, status=200)
def setup(self):
self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin')
self.container1_admin = core_factories.User()
self.container1_member = core_factories.User()
self.container1 = factories.DataContainer(users=[
{
"name": self.container1_admin["name"],
"capacity": "admin"
},
{
"name": self.container1_member["name"],
"capacity": "member"
},
])
self.dataset1 = factories.Dataset(owner_org=self.container1["id"])
self.resource1 = factories.Resource(
package_id=self.dataset1['id'],
upload=mocks.FakeFileStorage(),
visibility="restricted",
url="http://fakeurl/test.txt",
url_type='upload',
)
create_curation_activity('dataset_approved',
self.dataset1['id'],
self.dataset1['name'],
self.sysadmin['id'],
message='asdf')
create_download_activity({'user': self.sysadmin['name']},
self.resource1['id'])
toolkit.get_action('activity_create')({
'ignore_auth': True
}, {
'user_id': self.sysadmin['name'],
'object_id': self.dataset1['id'],
'activity_type': 'changed package',
'data': {}
})
def test_publish_dataset_with_resources(self, app):
draft_dataset = factories.Dataset(
name='dataset2',
title='Test Dataset 2',
owner_org='container1',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
operational_purpose_of_data='cartography',
user=self.user1,
state='draft',
)
factories.Resource(
name='resource1',
package_id=draft_dataset['name'],
url_type='upload',
visibility='restricted',
upload=mocks.FakeFileStorage(),
)
self.make_dataset_publish_request(
app,
draft_dataset['id'],
user='******',
status=200,
)
def setup(self):
self.target = factories.DataContainer(id='data-target', title='Target')
self.deposit_container = factories.DataContainer(
id='data-deposit',
name='data-deposit',
)
self.deposit_dataset = factories.DepositedDataset(
owner_org='data-deposit',
owner_org_dest=self.target['id'],
title='deposit',
tags=[{
'name': 'foobar'
}],
keywords=[KEYWORDS['Water Sanitation Hygiene']],
)
self.deposit_resource = factories.Resource(
package_id=self.deposit_dataset['id'],
url_type='upload',
)
self.sysadmin = core_factories.Sysadmin()
self.containers = [
factories.DataContainer(title='Container1'),
factories.DataContainer(title='Container1')
]
self.users = [
core_factories.User(fullname='User1'),
core_factories.User(fullname='User2'),
]
self.datasets = [
factories.Dataset(
user=self.sysadmin,
tags=[{
'name': 'economy'
}, {
'name': 'health'
}, {
'name': 'environment'
}],
keywords=[
KEYWORDS['Protection'], KEYWORDS['Food Security'],
KEYWORDS['Emergency Shelter and NFI']
],
owner_org=self.containers[0]['id'],
),
factories.Dataset(
user=self.users[0],
tags=[{
'name': 'health'
}, {
'name': 'environment'
}],
keywords=[
KEYWORDS['Food Security'],
KEYWORDS['Emergency Shelter and NFI']
],
owner_org=self.containers[0]['id'],
),
factories.Dataset(
user=self.users[0],
tags=[{
'name': 'health'
}],
keywords=[KEYWORDS['Food Security']],
owner_org=self.containers[0]['id'],
),
factories.Dataset(
user=self.users[0],
keywords=[KEYWORDS['Food Security']],
owner_org=self.containers[1]['id'],
),
]
self.resources = [
factories.Resource(package_id=self.datasets[0]['id'],
url_type='upload'),
factories.Resource(package_id=self.datasets[0]['id'],
url_type='upload'),
factories.Resource(package_id=self.datasets[1]['id'],
url_type='upload'),
factories.Resource(package_id=self.datasets[2]['id'],
url_type='upload'),
]
create_download_activity({'user': self.sysadmin['name']},
self.resources[0]['id'])
create_download_activity({'user': self.users[0]['name']},
self.resources[1]['id'])
create_download_activity({'user': self.sysadmin['name']},
self.resources[2]['id'])
create_download_activity({'user': self.users[0]['name']},
self.resources[2]['id'])
create_download_activity({'user': self.users[0]['name']},
self.resources[2]['id'])
create_download_activity({'user': self.sysadmin['name']},
self.deposit_resource['id'])