def setup(self, get_total_submissions, create_kobo_resources): get_total_submissions.return_value = 5 # any, > 0 self.kobo_user = factories.InternalKoBoUser() self.internal_user = factories.InternalUser() self.data_container = factories.DataContainer(users=[ { 'name': self.kobo_user['name'], 'capacity': 'editor' }, { 'name': self.internal_user['name'], 'capacity': 'member' }, ]) self.kobo_test_asset_id = 'test_kobo_id_1234' self.kobo_dataset = factories.Dataset( name='kobo-dataset', title='KoBo Dataset', owner_org=self.data_container['name'], data_collection_technique='f2f', sampling_procedure='nonprobability', operational_purpose_of_data='cartography', user=self.kobo_user, kobo_asset_id=self.kobo_test_asset_id, ) base_resource = dict(name='kobo-resource', package_id='kobo-dataset', url_type='upload', visibility='restricted', upload=mocks.FakeFileStorage(), url='original-file.csv', format='csv', kobo_type='data', kobo_details={ 'kobo_export_id': 'test_export_id', 'kobo_asset_id': self.kobo_test_asset_id, 'kobo_download_status': 'pending', 'kobo_download_attempts': 1, 'kobo_submission_count': 3, 'kobo_last_updated': datetime.datetime.utcnow().isoformat(), }) self.kobo_resource = factories.Resource(**base_resource) base_resource['kobo_details']['kobo_download_attempts'] = 5 base_resource['upload'] = mocks.FakeFileStorage() self.kobo_resource_last_attempt = factories.Resource(**base_resource)
def test_edit_kobo_resource_must_preserve_upload(self, submissions, create_kobo_resources, app): # required to create KoBo dataset submissions.return_value = 1 self.kobo_dataset = factories.Dataset( name='kobo-dataset', title='KoBo Dataset', owner_org='container1', data_collection_technique='f2f', sampling_procedure='nonprobability', operational_purpose_of_data='cartography', user=self.kobo_user, kobo_asset_id='test_1234', ) self.kobo_resource = factories.Resource(name='kobo-resource', package_id='kobo-dataset', url_type='upload', visibility='restricted', upload=mocks.FakeFileStorage(), url='original-file.csv', kobo_type='data') url = toolkit.url_for('resource.edit', id=self.kobo_dataset['id'], resource_id=self.kobo_resource['id']) env = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')} # Mock a resource edit payload form_data = { 'description': 'updated', 'url_type': 'upload', 'upload': mocks.FakeFileStorage(filename='different-file.csv'), # this is manually added to the update form as hidden fields: 'original_url': self.kobo_resource['url'], 'kobo_type': 'data', 'url': 'different-file.csv', 'clear_upload': '', 'save': '' } data = dict(self.kobo_resource, **form_data) resp = app.post(url, data=data, extra_environ=env, status=200) assert 'The form contains invalid entries:' in resp.body assert 'You cannot update a KoBoToolbox data file directly, please re-import the data instead' in resp.body
def setup(self): # Users self.normal_user = core_factories.User() self.org_user = core_factories.User() self.sysadmin = core_factories.Sysadmin() # Containers self.container = factories.DataContainer( users=[ {'name': self.org_user['name'], 'capacity': 'member'}, ] ) # Datasets self.dataset = factories.Dataset( owner_org=self.container['id'], ) self.resource = factories.Resource( package_id=self.dataset['id'], url_type='datastore', upload=mocks.FakeFileStorage(), visibility='restricted', ) # Actions core_helpers.call_action('datastore_create', resource_id=self.resource['id'], records=[{'a': 1, 'b': 2}], force=True, )
def test_upload_present_after_update(self): dataset = factories.Dataset() resource = factories.Resource( package_id=dataset['id'], upload=mocks.FakeFileStorage(), url = "http://fakeurl/test.txt", url_type='upload', ) resource['name'] = 'updated' updated_resource = core_helpers.call_action( 'resource_update', {'ignore_auth': True}, **resource ) assert updated_resource['name'] == 'updated' assert ( updated_resource['url'] == '{}/dataset/{}/resource/{}/download/test.txt'.format( toolkit.config['ckan.site_url'].rstrip('/'), dataset['id'], resource['id'] ) )
def setup(self): # Users self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin') self.user1 = core_factories.User(name='user1', id='user1') self.user2 = core_factories.User(name='user2', id='user2') self.user3 = core_factories.User(name='user3', id='user3') # Containers self.container1 = factories.DataContainer( name='container1', id='container1', users=[ { 'name': 'user1', 'capacity': 'admin' }, ], ) self.container2 = factories.DataContainer( name='container2', id='container2', users=[ { 'name': 'user2', 'capacity': 'admin' }, ], ) # Datasets self.dataset1 = factories.Dataset( name='dataset1', title='Test Dataset 1', owner_org='container1', data_collection_technique='f2f', sampling_procedure='nonprobability', operational_purpose_of_data='cartography', user=self.user1, tags=[{ 'name': 'Keyword1' }, { 'name': 'Keyword2' }], ) # Resources self.resource1 = factories.Resource( name='resource1', package_id='dataset1', url_type='upload', visibility='restricted', upload=mocks.FakeFileStorage(), )
def setup(self): self.internal_user = core_factories.User() self.dataset = factories.Dataset( user=self.internal_user ) self.resource = factories.Resource( package_id=self.dataset['id'], description='some description', visibility='public', identifiability='personally_identifiable', upload=mocks.FakeFileStorage(), url="http://fakeurl/test.txt", url_type='upload', ) self.resource_visible = factories.Resource( package_id=self.dataset['id'], description='some description 2', visibility='public', identifiability='anonymized_public', upload=mocks.FakeFileStorage(), url="http://fakeurl/test2.txt", url_type='upload', ) # Attachment type files do not use "identifiability" self.resource_ambiguous = factories.Resource( package_id=self.dataset['id'], description='some description 3', visibility='public', identifiability='personally_identifiable', upload=mocks.FakeFileStorage(), url="http://fakeurl/test3.txt", url_type='upload', type='attachment', file_type='other', )
def test_visibility_private(self, app): dataset = factories.Dataset( private=True, visibility='private', owner_org=self.container['id'], ) resource = factories.Resource( package_id=dataset['id'], url_type='upload', visibility='restricted', upload=mocks.FakeFileStorage(), ) # Even though we tried to save this with visibility='private' # this feature is disabled, so the validator has altered it for us # from now on the dataset should behave as 'restricted' assert dataset['visibility'] == 'restricted' resource_download_url = toolkit.url_for('resource.download', id=dataset['id'], resource_id=resource['id']) dataset_read_url = toolkit.url_for('dataset.read', id=dataset['id']) with app.flask_app.test_request_context(): environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')} res = app.get(dataset_read_url, extra_environ=environ, status=200) assert ( 'You are not authorized to download the private resources from this dataset' in res.body) app.get(resource_download_url, extra_environ=environ, status=403) with app.flask_app.test_request_context(): environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')} res = app.get(dataset_read_url, extra_environ=environ, status=200) assert ( 'You are not authorized to download the private resources from this dataset' not in res.body) app.get(resource_download_url, extra_environ=environ, status=200) with app.flask_app.test_request_context(): environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')} res = app.get(dataset_read_url, extra_environ=environ, status=200) assert ( 'You are not authorized to download the private resources from this dataset' not in res.body) app.get(resource_download_url, extra_environ=environ, status=200)
def test_upload_present(self): dataset = factories.Dataset() resource = factories.Resource( package_id=dataset['id'], upload=mocks.FakeFileStorage(), url = "http://fakeurl/test.txt", url_type='upload', ) assert ( resource['url'] == '{}/dataset/{}/resource/{}/download/test.txt'.format( toolkit.config.get('ckan.site_url').rstrip('/'), dataset['id'], resource['id'] ) )
def test_both_types_data_fields_missing(self): """ Test we have one error in the second spot of the error list Requires 2 resources. Only one invalid. """ dataset = factories.Dataset() resource1 = { 'name': 'Test File attachment', 'url': 'http://example.com/doc.pdf', 'format': 'PDF', 'description': 'Some description', 'type': 'attachment', 'file_type': 'report', 'visibility': 'public' } resource2 = { 'name': 'Test Data file', 'upload': mocks.FakeFileStorage(), 'format': 'CSV', 'description': 'Some data file', 'type': 'data', 'file_type': 'microdata', } dataset['resources'] = [resource1, resource2] with pytest.raises(toolkit.ValidationError) as e: call_action('package_update', {'user': self.sysadmin['name']}, **dataset) # First resource it's ok assert e.value.error_dict['resources'][0] == {} errors = e.value.error_dict['resources'][1] # TODO add 'visibility' after merge #583 for field in [ 'identifiability', 'date_range_end', 'version', 'date_range_start', 'process_status' ]: error = errors[field] assert error == ['Missing value']
def setup(self): self.sysadmin = core_factories.Sysadmin(name='sysadmin', id='sysadmin') self.container1_admin = core_factories.User() self.container1_member = core_factories.User() self.container1 = factories.DataContainer(users=[ { "name": self.container1_admin["name"], "capacity": "admin" }, { "name": self.container1_member["name"], "capacity": "member" }, ]) self.dataset1 = factories.Dataset(owner_org=self.container1["id"]) self.resource1 = factories.Resource( package_id=self.dataset1['id'], upload=mocks.FakeFileStorage(), visibility="restricted", url="http://fakeurl/test.txt", url_type='upload', ) create_curation_activity('dataset_approved', self.dataset1['id'], self.dataset1['name'], self.sysadmin['id'], message='asdf') create_download_activity({'user': self.sysadmin['name']}, self.resource1['id']) toolkit.get_action('activity_create')({ 'ignore_auth': True }, { 'user_id': self.sysadmin['name'], 'object_id': self.dataset1['id'], 'activity_type': 'changed package', 'data': {} })
def test_visibility_restricted(self, app): dataset = factories.Dataset(owner_org=self.container['id'], ) resource = factories.Resource( package_id=dataset['id'], url_type='upload', visibility='restricted', upload=mocks.FakeFileStorage(), ) resource_download_url = toolkit.url_for('resource.download', id=dataset['id'], resource_id=resource['id']) dataset_read_url = toolkit.url_for('dataset.read', id=dataset['id']) with app.flask_app.test_request_context(): environ = {'REMOTE_USER': self.normal_user['name'].encode('ascii')} res = app.get(dataset_read_url, extra_environ=environ, status=200) assert ( 'You are not authorized to download the private resources from this dataset' in res.body) app.get(resource_download_url, extra_environ=environ, status=403) with app.flask_app.test_request_context(): environ = {'REMOTE_USER': self.org_user['name'].encode('ascii')} res = app.get(dataset_read_url, extra_environ=environ, status=200) assert ( 'You are not authorized to download the private resources from this dataset' not in res.body) app.get(resource_download_url, extra_environ=environ, status=200) with app.flask_app.test_request_context(): environ = {'REMOTE_USER': self.sysadmin['name'].encode('ascii')} res = app.get(dataset_read_url, extra_environ=environ, status=200) assert ( 'You are not authorized to download the private resources from this dataset' not in res.body) app.get(resource_download_url, extra_environ=environ, status=200)
def test_publish_dataset_with_resources(self, app): draft_dataset = factories.Dataset( name='dataset2', title='Test Dataset 2', owner_org='container1', data_collection_technique='f2f', sampling_procedure='nonprobability', operational_purpose_of_data='cartography', user=self.user1, state='draft', ) factories.Resource( name='resource1', package_id=draft_dataset['name'], url_type='upload', visibility='restricted', upload=mocks.FakeFileStorage(), ) self.make_dataset_publish_request( app, draft_dataset['id'], user='******', status=200, )