def test_new_form_is_rendered(self, app): url = url_for('harvest_new') response = app.get(url, extra_environ=self.extra_environ) _assert_in_body('<form id="source-new"', response)
def test_resource_upload_then_clear(self): '''Test that clearing an upload removes the S3 key''' sysadmin = factories.Sysadmin(apikey="my-test-key") app = self._get_test_app() demo = ckanapi.TestAppCKAN(app, apikey='my-test-key') dataset = factories.Dataset(name="my-dataset") file_path = os.path.join(os.path.dirname(__file__), 'data.csv') resource = demo.action.resource_create(package_id='my-dataset', upload=open(file_path), url='file.txt') key = '{1}/resources/{0}/data.csv' \ .format(resource['id'], config.get('ckanext.s3filestore.aws_storage_path')) conn = boto.connect_s3() bucket = conn.get_bucket('my-bucket') # test the key exists assert_true(bucket.lookup(key)) # clear upload url = toolkit.url_for(controller='package', action='resource_edit', id=dataset['id'], resource_id=resource['id']) env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} app.post(url, {'clear_upload': True, 'url': 'http://asdf', 'save': 'save'}, extra_environ=env) # key shouldn't exist assert_false(bucket.lookup(key))
def get_validation_badge(resource, in_listing=False): if in_listing and not asbool( config.get('ckanext.validation.show_badges_in_listings', True)): return '' if not resource.get('validation_status'): return '' messages = { 'success': _('Valid data'), 'failure': _('Invalid data'), 'error': _('Error during validation'), 'unknown': _('Data validation unknown'), } if resource['validation_status'] in ['success', 'failure', 'error']: status = resource['validation_status'] else: status = 'unknown' validation_url = url_for('validation_read', id=resource['package_id'], resource_id=resource['id']) badge_url = url_for_static( '/images/badges/data-{}-flat.svg'.format(status)) return u''' <a href="{validation_url}" class="validation-badge"> <img src="{badge_url}" alt="{alt}" title="{title}"/> </a>'''.format(validation_url=validation_url, badge_url=badge_url, alt=messages[status], title=resource.get('validation_timestamp', ''))
def open_datarequest(id): data_dict = {'id': id} context = _get_context() # Basic initialization c.datarequest = {} try: check_access(OPEN_DATAREQUEST, context, data_dict) c.datarequest = get_action(SHOW_DATAREQUEST)(context, data_dict) if c.datarequest.get('closed', False) is False: return abort(403, _('This data request is already open')) else: data_dict = {} data_dict['id'] = id data_dict['organization_id'] = c.datarequest.get('organization_id') get_action(OPEN_DATAREQUEST)(context, data_dict) return redirect_to(url_for('datarequest.show', id=data_dict['id'])) except ValidationError as e: log.warn(e) errors_summary = _get_errors_summary(e.error_dict) return abort(403, errors_summary) except ObjectNotFound as e: log.warn(e) return abort(404, _('Data Request %s not found') % id) except NotAuthorized as e: log.warn(e) return abort( 403, _('You are not authorized to open the Data Request %s' % id))
def resource_download(self, id, resource_id, filename=None): ''' Provide a download by either redirecting the user to the url stored or downloading the uploaded file from S3. ''' context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if 'url' not in rsc: abort(404, _('No download is available')) elif rsc.get('url_type') == 'upload': upload = uploader.get_resource_uploader(rsc) bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') if filename is None: filename = os.path.basename(rsc['url']) key_path = upload.get_path(rsc['id'], filename) if filename is None: log.warn("Key '%s' not found in bucket '%s'", key_path, bucket_name) try: url = upload.get_signed_url_to_key(key_path) redirect(url) except ClientError as ex: if ex.response['Error']['Code'] in ['NoSuchKey', '404']: # attempt fallback if config.get( 'ckanext.s3filestore.filesystem_download_fallback', False): log.info( 'Attempting filesystem fallback for resource %s', resource_id) url = toolkit.url_for( controller= 'ckanext.s3filestore.controller:S3Controller', action='filesystem_resource_download', id=id, resource_id=resource_id, filename=filename) redirect(url) abort(404, _('Resource data not found')) else: raise ex redirect(rsc['url'])
def test_edit_form_is_rendered(self, app): source = harvest_factories.HarvestSource() url = url_for('harvest_edit', id=source['id']) response = app.get(url, extra_environ=self.extra_environ) _assert_in_body('<form id="source-new"', response)
def test_job_show_last_page_rendered(self, app): job = harvest_factories.HarvestJob() url = url_for('harvest_job_show_last', source=job['source_id']) response = app.get(url, extra_environ=self.extra_environ) _assert_in_body(job['id'], response)
def test_about_page_rendered(self, app): source = harvest_factories.HarvestSource() url = url_for('harvest_about', id=source['name']) response = app.get(url, extra_environ=self.extra_environ) _assert_in_body(source['name'], response)
def test_widget_loaded(app): dataset = factories.Dataset() for i in range(1, 11): resource = factories.Resource( name='Resource {}'.format(i), url='https://example.com', package_id=dataset['id'], ) if check_ckan_version(min_version='2.9.0'): url = url_for('dataset.read', id=dataset['id']) else: url = url_for(controller='package', action='read', id=dataset['id']) res = app.get(url) assert 'resources-list-filter-container' in res.get_data(as_text=True) assert 'script type="text/template" id="resources-list-template"' in res.get_data(as_text=True)
def test_admin_page_rendered(self, app): source_obj = harvest_factories.HarvestSourceObj() job = harvest_factories.HarvestJob(source=source_obj) url = url_for('harvest_admin', id=source_obj.id) response = app.get(url, extra_environ=self.extra_environ) _assert_in_body(source_obj.title, response) _assert_in_body(job['id'], response)
def test_job_show_last_page_rendered(self, app): job = harvest_factories.HarvestJob() sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} url = url_for('harvest_job_show_last', source=job['source_id']) response = app.get(url, extra_environ=env) _assert_in_body(job['id'], response)
def test_resource_upload_then_clear(self): '''Test that clearing an upload removes the S3 key''' sysadmin = factories.Sysadmin(apikey="my-test-key") app = self._get_test_app() demo = ckanapi.TestAppCKAN(app, apikey='my-test-key') dataset = factories.Dataset(name="my-dataset") file_path = os.path.join(os.path.dirname(__file__), 'data.csv') resource = demo.action.resource_create(package_id='my-dataset', upload=open(file_path), url='file.txt') key = '{1}/resources/{0}/data.csv' \ .format(resource['id'], config.get('ckanext.s3filestore.aws_storage_path')) s3 = self.botoSession.client('s3', endpoint_url=self.endpoint_url) # check whether the object exists in S3 # will throw exception if not existing s3.head_object(Bucket='my-bucket', Key=key) #conn = boto.connect_s3() #bucket = conn.get_bucket('my-bucket') # test the key exists #assert_true(bucket.lookup(key)) # clear upload url = toolkit.url_for(controller='package', action='resource_edit', id=dataset['id'], resource_id=resource['id']) env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} app.post(url, { 'clear_upload': True, 'url': 'http://asdf', 'save': 'save' }, extra_environ=env) # key shouldn't exist try: s3.head_object(Bucket='my-bucket', Key=key) assert_false(True, "file should not exist") except: # passed assert_true(True, "passed")
def test_admin_page_rendered(self, app): source_obj = harvest_factories.HarvestSourceObj() job = harvest_factories.HarvestJob(source=source_obj) sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} url = url_for('harvest_admin', id=source_obj.id) response = app.get(url, extra_environ=env) _assert_in_body(source_obj.title, response) _assert_in_body(job['id'], response)
def test_resource_upload_with_url_and_clear(self): '''Test that clearing an upload and using a URL does not crash''' sysadmin = factories.Sysadmin(apikey="my-test-key") app = self._get_test_app() dataset = factories.Dataset(name="my-dataset") url = toolkit.url_for(controller='package', action='new_resource', id=dataset['id']) env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} app.post(url, {'clear_upload': True, 'id': '', # Emtpy id from the form 'url': 'http://asdf', 'save': 'save'}, extra_environ=env)
def get_validation_badge(resource, in_listing=False): if in_listing and not asbool( config.get('ckanext.validation.show_badges_in_listings', True)): return '' if not resource.get('validation_status'): return '' statuses = { 'success': _('success'), 'failure': _('failure'), 'invalid': _('invalid'), 'error': _('error'), 'unknown': _('unknown'), } if resource['validation_status'] in ['success', 'failure', 'error']: status = resource['validation_status'] if status == 'failure': status = 'invalid' else: status = 'unknown' if check_ckan_version(min_version='2.9.0'): action = 'validation.read' else: action = 'validation_read' validation_url = url_for( action, id=resource['package_id'], resource_id=resource['id']) return u''' <a href="{validation_url}" class="validation-badge" title="{title}"> <span class="prefix">{prefix}</span><span class="status {status}">{status_title}</span> </a>'''.format( validation_url=validation_url, prefix=_('data'), status=status, status_title=statuses[status], title=resource.get('validation_timestamp', ''))
def test_resource_upload_with_url_and_clear(self): '''Test that clearing an upload and using a URL does not crash''' sysadmin = factories.Sysadmin(apikey="my-test-key") app = helpers._get_test_app() dataset = factories.Dataset(name="my-dataset") url = toolkit.url_for(controller='package', action='new_resource', id=dataset['id']) env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} app.post( url, { 'clear_upload': True, 'id': '', # Empty id from the form 'url': 'http://asdf', 'save': 'save' }, extra_environ=env)
def resource_download(self, id, resource_id, filename=None): ''' Provide a download by either redirecting the user to the url stored or downloading the uploaded file from S3. ''' context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.get_resource_uploader(rsc) bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') region = config.get('ckanext.s3filestore.region_name') host_name = config.get('ckanext.s3filestore.host_name') bucket = upload.get_s3_bucket(bucket_name) if filename is None: filename = os.path.basename(rsc['url']) key_path = upload.get_path(rsc['id'], filename) key = filename if key is None: log.warn('Key \'{0}\' not found in bucket \'{1}\''.format( key_path, bucket_name)) try: # Small workaround to manage downloading of large files # We are using redirect to minio's resource public URL s3 = upload.get_s3_session() client = s3.client(service_name='s3', endpoint_url=host_name) url = client.generate_presigned_url(ClientMethod='get_object', Params={ 'Bucket': bucket.name, 'Key': key_path }, ExpiresIn=60) redirect(url) except ClientError as ex: if ex.response['Error']['Code'] == 'NoSuchKey': # attempt fallback if config.get( 'ckanext.s3filestore.filesystem_download_fallback', False): log.info( 'Attempting filesystem fallback for resource {0}'. format(resource_id)) url = toolkit.url_for( controller= 'ckanext.s3filestore.controller:S3Controller', action='filesystem_resource_download', id=id, resource_id=resource_id, filename=filename) redirect(url) abort(404, _('Resource data not found')) else: raise ex
def resource_download(self, id, resource_id, filename=None): ''' Provide a download by either redirecting the user to the url stored or downloading the uploaded file from S3. ''' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.get_resource_uploader(rsc) bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') region = config.get('ckanext.s3filestore.region_name') host_name = config.get('ckanext.s3filestore.host_name') bucket = upload.get_s3_bucket(bucket_name) if filename is None: filename = os.path.basename(rsc['url']) key_path = upload.get_path(rsc['id'], filename) key = filename if key is None: log.warn('Key \'{0}\' not found in bucket \'{1}\'' .format(key_path, bucket_name)) try: # Small workaround to manage downloading of large files # We are using redirect to minio's resource public URL s3 = upload.get_s3_session() client = s3.client(service_name='s3', endpoint_url=host_name) url = client.generate_presigned_url(ClientMethod='get_object', Params={'Bucket': bucket.name, 'Key': key_path}, ExpiresIn=60) redirect(url) except ClientError as ex: if ex.response['Error']['Code'] == 'NoSuchKey': # attempt fallback if config.get( 'ckanext.s3filestore.filesystem_download_fallback', False): log.info('Attempting filesystem fallback for resource {0}' .format(resource_id)) url = toolkit.url_for( controller='ckanext.s3filestore.controller:S3Controller', action='filesystem_resource_download', id=id, resource_id=resource_id, filename=filename) redirect(url) abort(404, _('Resource data not found')) else: raise ex
def resource_download(package_type, id, resource_id, filename=None): ''' Provide a download by either redirecting the user to the url stored or downloading the uploaded file from S3. ''' context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: return abort(404, _('Resource not found')) except NotAuthorized: return abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.get_resource_uploader(rsc) preview = request.args.get(u'preview', False) if filename is None: filename = os.path.basename(rsc['url']) key_path = upload.get_path(rsc['id'], filename) key = filename if key is None: log.warn('Key \'{0}\' not found in bucket \'{1}\''.format( key_path, upload.bucket_name)) try: if rsc.get('mimetype') == 'text/html' and preview: url = upload.get_signed_url_to_key(key_path) else: params = { 'ResponseContentDisposition': 'attachment; filename=' + filename, } url = upload.get_signed_url_to_key(key_path, params) return redirect(url) except ClientError as ex: if ex.response['Error']['Code'] in ['NoSuchKey', '404']: # attempt fallback if ckan_config.get( 'ckanext.s3filestore.filesystem_download_fallback', False): log.info('Attempting filesystem fallback for resource {0}'. format(resource_id)) url = toolkit.url_for( u's3_resource.filesystem_resource_download', id=id, resource_id=resource_id, filename=filename, preview=preview) return redirect(url) return abort(404, _('Resource data not found')) else: raise ex else: return redirect(rsc[u'url'])