def test_new_form_is_rendered(self, app):
url = url_for('harvest_new')
response = app.get(url, extra_environ=self.extra_environ)
_assert_in_body('<form id="source-new"', response)
def test_resource_upload_then_clear(self):
'''Test that clearing an upload removes the S3 key'''
sysadmin = factories.Sysadmin(apikey="my-test-key")
app = self._get_test_app()
demo = ckanapi.TestAppCKAN(app, apikey='my-test-key')
dataset = factories.Dataset(name="my-dataset")
file_path = os.path.join(os.path.dirname(__file__), 'data.csv')
resource = demo.action.resource_create(package_id='my-dataset',
upload=open(file_path),
url='file.txt')
key = '{1}/resources/{0}/data.csv' \
.format(resource['id'],
config.get('ckanext.s3filestore.aws_storage_path'))
conn = boto.connect_s3()
bucket = conn.get_bucket('my-bucket')
# test the key exists
assert_true(bucket.lookup(key))
# clear upload
url = toolkit.url_for(controller='package', action='resource_edit',
id=dataset['id'], resource_id=resource['id'])
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
app.post(url, {'clear_upload': True,
'url': 'http://asdf', 'save': 'save'},
extra_environ=env)
# key shouldn't exist
assert_false(bucket.lookup(key))
def get_validation_badge(resource, in_listing=False):
if in_listing and not asbool(
config.get('ckanext.validation.show_badges_in_listings', True)):
return ''
if not resource.get('validation_status'):
return ''
messages = {
'success': _('Valid data'),
'failure': _('Invalid data'),
'error': _('Error during validation'),
'unknown': _('Data validation unknown'),
}
if resource['validation_status'] in ['success', 'failure', 'error']:
status = resource['validation_status']
else:
status = 'unknown'
validation_url = url_for('validation_read',
id=resource['package_id'],
resource_id=resource['id'])
badge_url = url_for_static(
'/images/badges/data-{}-flat.svg'.format(status))
return u'''
<a href="{validation_url}" class="validation-badge">
<img src="{badge_url}" alt="{alt}" title="{title}"/>
</a>'''.format(validation_url=validation_url,
badge_url=badge_url,
alt=messages[status],
title=resource.get('validation_timestamp', ''))
def open_datarequest(id):
data_dict = {'id': id}
context = _get_context()
# Basic initialization
c.datarequest = {}
try:
check_access(OPEN_DATAREQUEST, context, data_dict)
c.datarequest = get_action(SHOW_DATAREQUEST)(context, data_dict)
if c.datarequest.get('closed', False) is False:
return abort(403, _('This data request is already open'))
else:
data_dict = {}
data_dict['id'] = id
data_dict['organization_id'] = c.datarequest.get('organization_id')
get_action(OPEN_DATAREQUEST)(context, data_dict)
return redirect_to(url_for('datarequest.show', id=data_dict['id']))
except ValidationError as e:
log.warn(e)
errors_summary = _get_errors_summary(e.error_dict)
return abort(403, errors_summary)
except ObjectNotFound as e:
log.warn(e)
return abort(404, _('Data Request %s not found') % id)
except NotAuthorized as e:
log.warn(e)
return abort(
403, _('You are not authorized to open the Data Request %s' % id))
def resource_download(self, id, resource_id, filename=None):
'''
Provide a download by either redirecting the user to the url stored or
downloading the uploaded file from S3.
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
try:
rsc = get_action('resource_show')(context, {'id': resource_id})
get_action('package_show')(context, {'id': id})
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
if 'url' not in rsc:
abort(404, _('No download is available'))
elif rsc.get('url_type') == 'upload':
upload = uploader.get_resource_uploader(rsc)
bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
if filename is None:
filename = os.path.basename(rsc['url'])
key_path = upload.get_path(rsc['id'], filename)
if filename is None:
log.warn("Key '%s' not found in bucket '%s'", key_path,
bucket_name)
try:
url = upload.get_signed_url_to_key(key_path)
redirect(url)
except ClientError as ex:
if ex.response['Error']['Code'] in ['NoSuchKey', '404']:
# attempt fallback
if config.get(
'ckanext.s3filestore.filesystem_download_fallback',
False):
log.info(
'Attempting filesystem fallback for resource %s',
resource_id)
url = toolkit.url_for(
controller=
'ckanext.s3filestore.controller:S3Controller',
action='filesystem_resource_download',
id=id,
resource_id=resource_id,
filename=filename)
redirect(url)
abort(404, _('Resource data not found'))
else:
raise ex
redirect(rsc['url'])
def test_edit_form_is_rendered(self, app):
source = harvest_factories.HarvestSource()
url = url_for('harvest_edit', id=source['id'])
response = app.get(url, extra_environ=self.extra_environ)
_assert_in_body('<form id="source-new"', response)
def test_job_show_last_page_rendered(self, app):
job = harvest_factories.HarvestJob()
url = url_for('harvest_job_show_last', source=job['source_id'])
response = app.get(url, extra_environ=self.extra_environ)
_assert_in_body(job['id'], response)
def test_about_page_rendered(self, app):
source = harvest_factories.HarvestSource()
url = url_for('harvest_about', id=source['name'])
response = app.get(url, extra_environ=self.extra_environ)
_assert_in_body(source['name'], response)
def test_widget_loaded(app):
dataset = factories.Dataset()
for i in range(1, 11):
resource = factories.Resource(
name='Resource {}'.format(i),
url='https://example.com',
package_id=dataset['id'],
)
if check_ckan_version(min_version='2.9.0'):
url = url_for('dataset.read', id=dataset['id'])
else:
url = url_for(controller='package', action='read', id=dataset['id'])
res = app.get(url)
assert 'resources-list-filter-container' in res.get_data(as_text=True)
assert 'script type="text/template" id="resources-list-template"' in res.get_data(as_text=True)
def test_admin_page_rendered(self, app):
source_obj = harvest_factories.HarvestSourceObj()
job = harvest_factories.HarvestJob(source=source_obj)
url = url_for('harvest_admin', id=source_obj.id)
response = app.get(url, extra_environ=self.extra_environ)
_assert_in_body(source_obj.title, response)
_assert_in_body(job['id'], response)
def test_job_show_last_page_rendered(self, app):
job = harvest_factories.HarvestJob()
sysadmin = factories.Sysadmin()
env = {"REMOTE_USER": sysadmin['name'].encode('ascii')}
url = url_for('harvest_job_show_last', source=job['source_id'])
response = app.get(url, extra_environ=env)
_assert_in_body(job['id'], response)
def test_resource_upload_then_clear(self):
'''Test that clearing an upload removes the S3 key'''
sysadmin = factories.Sysadmin(apikey="my-test-key")
app = self._get_test_app()
demo = ckanapi.TestAppCKAN(app, apikey='my-test-key')
dataset = factories.Dataset(name="my-dataset")
file_path = os.path.join(os.path.dirname(__file__), 'data.csv')
resource = demo.action.resource_create(package_id='my-dataset',
upload=open(file_path),
url='file.txt')
key = '{1}/resources/{0}/data.csv' \
.format(resource['id'],
config.get('ckanext.s3filestore.aws_storage_path'))
s3 = self.botoSession.client('s3', endpoint_url=self.endpoint_url)
# check whether the object exists in S3
# will throw exception if not existing
s3.head_object(Bucket='my-bucket', Key=key)
#conn = boto.connect_s3()
#bucket = conn.get_bucket('my-bucket')
# test the key exists
#assert_true(bucket.lookup(key))
# clear upload
url = toolkit.url_for(controller='package',
action='resource_edit',
id=dataset['id'],
resource_id=resource['id'])
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
app.post(url, {
'clear_upload': True,
'url': 'http://asdf',
'save': 'save'
},
extra_environ=env)
# key shouldn't exist
try:
s3.head_object(Bucket='my-bucket', Key=key)
assert_false(True, "file should not exist")
except:
# passed
assert_true(True, "passed")
def test_admin_page_rendered(self, app):
source_obj = harvest_factories.HarvestSourceObj()
job = harvest_factories.HarvestJob(source=source_obj)
sysadmin = factories.Sysadmin()
env = {"REMOTE_USER": sysadmin['name'].encode('ascii')}
url = url_for('harvest_admin', id=source_obj.id)
response = app.get(url, extra_environ=env)
_assert_in_body(source_obj.title, response)
_assert_in_body(job['id'], response)
def test_resource_upload_with_url_and_clear(self):
'''Test that clearing an upload and using a URL does not crash'''
sysadmin = factories.Sysadmin(apikey="my-test-key")
app = self._get_test_app()
dataset = factories.Dataset(name="my-dataset")
url = toolkit.url_for(controller='package', action='new_resource',
id=dataset['id'])
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
app.post(url, {'clear_upload': True,
'id': '', # Emtpy id from the form
'url': 'http://asdf', 'save': 'save'},
extra_environ=env)
def get_validation_badge(resource, in_listing=False):
if in_listing and not asbool(
config.get('ckanext.validation.show_badges_in_listings', True)):
return ''
if not resource.get('validation_status'):
return ''
statuses = {
'success': _('success'),
'failure': _('failure'),
'invalid': _('invalid'),
'error': _('error'),
'unknown': _('unknown'),
}
if resource['validation_status'] in ['success', 'failure', 'error']:
status = resource['validation_status']
if status == 'failure':
status = 'invalid'
else:
status = 'unknown'
if check_ckan_version(min_version='2.9.0'):
action = 'validation.read'
else:
action = 'validation_read'
validation_url = url_for(
action,
id=resource['package_id'],
resource_id=resource['id'])
return u'''
<a href="{validation_url}" class="validation-badge" title="{title}">
<span class="prefix">{prefix}</span><span class="status {status}">{status_title}</span>
</a>'''.format(
validation_url=validation_url,
prefix=_('data'),
status=status,
status_title=statuses[status],
title=resource.get('validation_timestamp', ''))
def test_resource_upload_with_url_and_clear(self):
'''Test that clearing an upload and using a URL does not crash'''
sysadmin = factories.Sysadmin(apikey="my-test-key")
app = helpers._get_test_app()
dataset = factories.Dataset(name="my-dataset")
url = toolkit.url_for(controller='package',
action='new_resource',
id=dataset['id'])
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
app.post(
url,
{
'clear_upload': True,
'id': '', # Empty id from the form
'url': 'http://asdf',
'save': 'save'
},
extra_environ=env)
def resource_download(self, id, resource_id, filename=None):
'''
Provide a download by either redirecting the user to the url stored or
downloading the uploaded file from S3.
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
try:
rsc = get_action('resource_show')(context, {'id': resource_id})
get_action('package_show')(context, {'id': id})
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
if rsc.get('url_type') == 'upload':
upload = uploader.get_resource_uploader(rsc)
bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
region = config.get('ckanext.s3filestore.region_name')
host_name = config.get('ckanext.s3filestore.host_name')
bucket = upload.get_s3_bucket(bucket_name)
if filename is None:
filename = os.path.basename(rsc['url'])
key_path = upload.get_path(rsc['id'], filename)
key = filename
if key is None:
log.warn('Key \'{0}\' not found in bucket \'{1}\''.format(
key_path, bucket_name))
try:
# Small workaround to manage downloading of large files
# We are using redirect to minio's resource public URL
s3 = upload.get_s3_session()
client = s3.client(service_name='s3', endpoint_url=host_name)
url = client.generate_presigned_url(ClientMethod='get_object',
Params={
'Bucket': bucket.name,
'Key': key_path
},
ExpiresIn=60)
redirect(url)
except ClientError as ex:
if ex.response['Error']['Code'] == 'NoSuchKey':
# attempt fallback
if config.get(
'ckanext.s3filestore.filesystem_download_fallback',
False):
log.info(
'Attempting filesystem fallback for resource {0}'.
format(resource_id))
url = toolkit.url_for(
controller=
'ckanext.s3filestore.controller:S3Controller',
action='filesystem_resource_download',
id=id,
resource_id=resource_id,
filename=filename)
redirect(url)
abort(404, _('Resource data not found'))
else:
raise ex
def resource_download(self, id, resource_id, filename=None):
'''
Provide a download by either redirecting the user to the url stored or
downloading the uploaded file from S3.
'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj}
try:
rsc = get_action('resource_show')(context, {'id': resource_id})
get_action('package_show')(context, {'id': id})
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
if rsc.get('url_type') == 'upload':
upload = uploader.get_resource_uploader(rsc)
bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
region = config.get('ckanext.s3filestore.region_name')
host_name = config.get('ckanext.s3filestore.host_name')
bucket = upload.get_s3_bucket(bucket_name)
if filename is None:
filename = os.path.basename(rsc['url'])
key_path = upload.get_path(rsc['id'], filename)
key = filename
if key is None:
log.warn('Key \'{0}\' not found in bucket \'{1}\''
.format(key_path, bucket_name))
try:
# Small workaround to manage downloading of large files
# We are using redirect to minio's resource public URL
s3 = upload.get_s3_session()
client = s3.client(service_name='s3', endpoint_url=host_name)
url = client.generate_presigned_url(ClientMethod='get_object',
Params={'Bucket': bucket.name,
'Key': key_path},
ExpiresIn=60)
redirect(url)
except ClientError as ex:
if ex.response['Error']['Code'] == 'NoSuchKey':
# attempt fallback
if config.get(
'ckanext.s3filestore.filesystem_download_fallback',
False):
log.info('Attempting filesystem fallback for resource {0}'
.format(resource_id))
url = toolkit.url_for(
controller='ckanext.s3filestore.controller:S3Controller',
action='filesystem_resource_download',
id=id,
resource_id=resource_id,
filename=filename)
redirect(url)
abort(404, _('Resource data not found'))
else:
raise ex
def resource_download(package_type, id, resource_id, filename=None):
'''
Provide a download by either redirecting the user to the url stored or
downloading the uploaded file from S3.
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
try:
rsc = get_action('resource_show')(context, {'id': resource_id})
get_action('package_show')(context, {'id': id})
except NotFound:
return abort(404, _('Resource not found'))
except NotAuthorized:
return abort(401, _('Unauthorized to read resource %s') % id)
if rsc.get('url_type') == 'upload':
upload = uploader.get_resource_uploader(rsc)
preview = request.args.get(u'preview', False)
if filename is None:
filename = os.path.basename(rsc['url'])
key_path = upload.get_path(rsc['id'], filename)
key = filename
if key is None:
log.warn('Key \'{0}\' not found in bucket \'{1}\''.format(
key_path, upload.bucket_name))
try:
if rsc.get('mimetype') == 'text/html' and preview:
url = upload.get_signed_url_to_key(key_path)
else:
params = {
'ResponseContentDisposition':
'attachment; filename=' + filename,
}
url = upload.get_signed_url_to_key(key_path, params)
return redirect(url)
except ClientError as ex:
if ex.response['Error']['Code'] in ['NoSuchKey', '404']:
# attempt fallback
if ckan_config.get(
'ckanext.s3filestore.filesystem_download_fallback',
False):
log.info('Attempting filesystem fallback for resource {0}'.
format(resource_id))
url = toolkit.url_for(
u's3_resource.filesystem_resource_download',
id=id,
resource_id=resource_id,
filename=filename,
preview=preview)
return redirect(url)
return abort(404, _('Resource data not found'))
else:
raise ex
else:
return redirect(rsc[u'url'])
