def _get_homepage_views():
homepage_view_ids = [
view.resource_view_id
for view in db.Featured.find(homepage=True).all()
]
resource_views = model.Session.query(model.ResourceView).filter(
model.ResourceView.id.in_(homepage_view_ids)).all()
homepage_views = []
for view in resource_views:
resource_view = md.resource_view_dictize(view, {'model': model})
resource_obj = model.Resource.get(resource_view['resource_id'])
if resource_obj.state == 'deleted':
continue
resource = md.resource_dictize(resource_obj, {'model': model})
homepage_views.append({
'resource_view':
resource_view,
'resource':
resource,
'package':
md.package_dictize(resource_obj.package, {'model': model})
})
return homepage_views
def _create_archiver_task(self, resource):
from ckan.lib.base import c
site_user = get_action('get_site_user')(
{'model': model, 'ignore_auth': True, 'defer_commit': True}, {}
)
user = model.User.by_name(c.user)
context = json.dumps({
'site_url': self.site_url,
'apikey': user.apikey,
'username': user.name,
'cache_url_root': self.cache_url_root,
'site_user_apikey': site_user['apikey']
})
data = json.dumps(resource_dictize(resource, {'model': model}))
task_id = make_uuid()
archiver_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'archiver',
'key': u'celery_task_id',
'value': task_id,
'error': u'',
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'user': site_user['name'],
'ignore_auth': True
}
get_action('task_status_update')(archiver_task_context, archiver_task_status)
celery.send_task("archiver.update", args=[context, data], task_id=task_id)
def _create_datastorer_task(self, resource):
user = get_action('get_site_user')({'model': model,
'ignore_auth': True,
'defer_commit': True}, {})
context = json.dumps({
'site_url': h.url_for_static('/', qualified=True),
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'username': user.get('name'),
})
data = json.dumps(resource_dictize(resource, {'model': model}))
task_id = make_uuid()
datastorer_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'datastorer',
'key': u'celery_task_id',
'value': task_id,
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'user': user.get('name'),
}
get_action('task_status_update')(archiver_task_context,
datastorer_task_status)
celery.send_task("datastorer.upload",
args=[context, data],
task_id=task_id)
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'alt_url': u'empty resource group id',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'size': u'123',
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'state': u'active',
'url': u'test'
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
res = model.Session.query(model.Resource).filter_by(url=u'test').one()
res_dictized = self.remove_changable_columns(
resource_dictize(res, context))
assert res_dictized == new_resource, res_dictized
def identify_resource(resource):
# With resource_dictize we get the correct resource url
# even if dataset is in draft state
task_id = make_uuid()
resource_dict = resource_dictize(resource, {'model': model})
context = _make_default_context()
celery.send_task('vectorstorer.identify',
args=[resource_dict, context],
countdown=15,
task_id=task_id)
res_identify = model.Session.query(ResourceIngest).filter(
ResourceIngest.resource_id == resource.id).first()
if res_identify:
# This is when a user had previously rejected the ingestion workflow,
# but now wants to re-identify the resource
model.Session.delete(res_identify)
new_res_identify = ResourceIngest(task_id, resource.id,
ResourceStorerType.VECTOR)
model.Session.add(new_res_identify)
model.Session.commit()
else:
# A newly created/updated resource needs to be identified
new_res_identify = ResourceIngest(task_id, resource.id,
ResourceStorerType.VECTOR)
model.Session.add(new_res_identify)
def _create_datastorer_task(self, resource):
user = get_action('get_site_user')({'model': model,
'ignore_auth': True,
'defer_commit': True}, {})
if not hasattr(self, 'site_url'):
from pylons import config
self.site_url = config.get('ckan.site_url_internally') or \
config.get('ckan.site_url')
context = json.dumps({
'site_url': self.site_url,
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'username': user.get('name'),
})
data = json.dumps(resource_dictize(resource, {'model': model}))
task_id = make_uuid()
datastorer_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'datastorer',
'key': u'celery_task_id',
'value': task_id,
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'user': user.get('name'),
}
get_action('task_status_update')(archiver_task_context,
datastorer_task_status)
celery.send_task("datastorer.upload",
args=[context, data],
task_id=task_id)
def _get_broken_resource_links(organisation_id=None):
organisation_id = None
query = Session.query(Package.name, Package.title, PackageExtra.value, Resource)\
.join(PackageExtra)\
.join(ResourceGroup, Package.id==ResourceGroup.package_id)\
.join(Resource)\
.join(TaskStatus, TaskStatus.entity_id==Resource.id)\
.filter(TaskStatus.key==u'openness_score')\
.filter(TaskStatus.value==u'0')\
.filter(or_(
and_(PackageExtra.key=='published_by',
PackageExtra.value.like('%%[%s]' % (organisation_id is None and '%' or organisation_id))),
and_(PackageExtra.key=='published_via',
PackageExtra.value.like('%%[%s]' % (organisation_id is None and '%' or organisation_id))),
)\
)\
.distinct()
context = {'model': model, 'session': model.Session}
data = []
for row in query:
resource = resource_dictize(row.Resource, context)
task_data = {'entity_id': resource['id'], 'task_type': 'qa', 'key': 'openness_score_reason'}
status = get_action('task_status_show')(context, task_data)
resource['openness_score'] = u'0'
resource['openness_score_reason'] = status.get('value')
data.append([row.name, row.title, row.value, resource])
return _collapse(data, [_extract_publisher, _extract_dataset])
def _create_task(self, resource):
user = get_action('get_site_user')({'model': model,
'ignore_auth': True,
'defer_commit': True}, {})
context = json.dumps({
'site_url': self.site_url,
'apikey': user.get('apikey')
})
data = json.dumps(resource_dictize(resource, {'model': model}))
task_id = make_uuid()
task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'qa',
'key': u'celery_task_id',
'value': task_id,
'error': u'',
'last_updated': datetime.now().isoformat()
}
task_context = {
'model': model,
'user': user.get('name'),
}
get_action('task_status_update')(task_context, task_status)
celery.send_task("qa.update", args=[context, data], task_id=task_id)
def resource_update(context, data_dict):
model = context['model']
user = context['user']
id = data_dict["id"]
schema = context.get('schema') or ckan.logic.schema.default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
check_access('resource_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def _get_broken_resource_links(organisation_id=None):
organisation_id = None
query = Session.query(Package.name, Package.title, PackageExtra.value, Resource)\
.join(PackageExtra)\
.join(ResourceGroup, Package.id==ResourceGroup.package_id)\
.join(Resource)\
.join(TaskStatus, TaskStatus.entity_id==Resource.id)\
.filter(TaskStatus.key==u'openness_score')\
.filter(TaskStatus.value==u'0')\
.filter(or_(
and_(PackageExtra.key=='published_by',
PackageExtra.value.like('%%[%s]' % (organisation_id is None and '%' or organisation_id))),
and_(PackageExtra.key=='published_via',
PackageExtra.value.like('%%[%s]' % (organisation_id is None and '%' or organisation_id))),
)\
)\
.distinct()
context = {'model': model, 'session': model.Session}
data = []
for row in query:
resource = resource_dictize(row.Resource, context)
task_data = {
'entity_id': resource['id'],
'task_type': 'qa',
'key': 'openness_score_reason'
}
status = get_action('task_status_show')(context, task_data)
resource['openness_score'] = u'0'
resource['openness_score_reason'] = status.get('value')
data.append([row.name, row.title, row.value, resource])
return _collapse(data, [_extract_publisher, _extract_dataset])
def broken_resource_links_by_dataset():
"""
Return a list of named tuples, one for each dataset that contains
broken resource links (defined as resources with an openness score of 0).
The named tuple is of the form:
(name (str), title (str), resources (list of dicts))
"""
query = Session.query(Package.name, Package.title, Resource)\
.join(ResourceGroup, Package.id==ResourceGroup.package_id)\
.join(Resource)\
.join(TaskStatus, TaskStatus.entity_id==Resource.id)\
.filter(TaskStatus.key==u'openness_score')\
.filter(TaskStatus.value==u'0')\
.distinct()
context = {'model': model, 'session': model.Session}
results = {}
for name, title, resource in query:
resource = resource_dictize(resource, context)
data = {'entity_id': resource['id'], 'task_type': 'qa', 'key': 'openness_score_reason'}
status = get_action('task_status_show')(context, data)
resource['openness_score_reason'] = status.get('value')
if name in results:
results[name].resources.append(resource)
else:
DatasetTuple = namedtuple('DatasetTuple', ['name', 'title', 'resources'])
results[name] = DatasetTuple(name, title or name, [resource])
return results.values()
def _get_canonical_view(package_id):
canonical_view_ids = [
view.resource_view_id for view in db.Featured.find(
package_id=package_id, canonical=True).all()
]
if not canonical_view_ids:
return None
resource_views = model.Session.query(model.ResourceView).filter(
model.ResourceView.id.in_(canonical_view_ids)).all()
if resource_views is None:
return None
for view in resource_views:
resource_view = md.resource_view_dictize(view, {'model': model})
resource_obj = model.Resource.get(resource_view['resource_id'])
if resource_obj.state == 'deleted':
continue
resource = md.resource_dictize(resource_obj, {'model': model})
return {'resource': resource, 'resource_view': resource_view}
return None
def _create_task(self, resource):
user = get_action('get_site_user')({
'model': model,
'ignore_auth': True,
'defer_commit': True
}, {})
context = json.dumps({
'site_url': self.site_url,
'apikey': user.get('apikey')
})
data = json.dumps(resource_dictize(resource, {'model': model}))
task_id = make_uuid()
task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'qa',
'key': u'celery_task_id',
'value': task_id,
'error': u'',
'last_updated': datetime.now().isoformat()
}
task_context = {
'model': model,
'user': user.get('name'),
}
get_action('task_status_update')(task_context, task_status)
celery.send_task("qa.update", args=[context, data], task_id=task_id)
def resource_update(context, data_dict):
model = context["model"]
session = context["session"]
user = context["user"]
id = data_dict["id"]
schema = context.get("schema") or default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error("Could not find resource " + id)
raise NotFound(_("Resource was not found."))
check_access("resource_update", context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, resource_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if "message" in context:
rev.message = context["message"]
else:
rev.message = _(u"REST API: Update object %s") % data.get("name", "")
resource = resource_dict_save(data, context)
if not context.get("defer_commit"):
model.repo.commit()
return resource_dictize(resource, context)
def test_14_resource_no_id(self):
context = {"model": model,
"session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'alt_url': u'empty resource group id',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'size': u'123',
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'state': u'active',
'url': u'test'
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
res = model.Session.query(model.Resource).filter_by(url=u'test').one()
res_dictized = self.remove_changable_columns(resource_dictize(res, context))
assert res_dictized == new_resource, res_dictized
def after_update(self, context, resource):
otype = resource.get('type')
lang = helpers.getLanguage()
if otype != 'dataset' and lang:
# r = model.Session.query(model.Resource).filter(model.Resource.id == resource.get('id')).all()
r = model.Resource.get(resource.get('id'))
if r:
r = model_dictize.resource_dictize(r, {
'model': model,
'session': model.Session
})
# MULTILANG - persisting resource localized record in multilang table
# q_results = model.Session.query(ResourceMultilang).filter(ResourceMultilang.resource_id == resource.get('id'), ResourceMultilang.lang == lang).all()
q_results = ResourceMultilang.get_for_resource_id_and_lang(
r.get('id'), lang)
if q_results and q_results.count() > 0:
for result in q_results:
result.text = r.get(result.field)
result.save()
else:
log.info(
'Localised fields are missing in resource_multilang table, persisting ...'
)
ResourceMultilang.persist(r, lang)
def _create_webstorer_task(self, resource):
user = get_action('get_site_user')({'model': model,
'ignore_auth': True,
'defer_commit': True}, {})
context = json.dumps({
'site_url': self.site_url,
'apikey': user.get('apikey'),
'username': user.get('name'),
'webstore_url': self.webstore_url
})
data = json.dumps(resource_dictize(resource, {'model': model}))
webstorer_task = send_task("webstorer.upload", [context, data])
# update the task_status table
webstorer_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'webstorer',
'key': u'celery_task_id',
'value': webstorer_task.task_id,
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'session': model.Session,
'user': user.get('name'),
'defer_commit': True
}
get_action('task_status_update')(archiver_task_context, webstorer_task_status)
def _get_organizationpage_views():
organizationpage_view_ids = []
organizationpage_views = []
# list out all the resource ID whose is featured with package IDs in the organization
try:
resp = c.page.items
if len(resp)>0 and resp[0].has_key('id'):
for items in resp:
pkg_id = items['id']
for view in db.Civicdata_Featured.find(organizationpage=True, package_id=pkg_id).all():
organizationpage_view_ids.append(view.resource_view_id)
resource_views = model.Session.query(model.ResourceView).filter(model.ResourceView.id.in_(organizationpage_view_ids)).all()
for view in resource_views:
resource_view = md.resource_view_dictize(view, {'model': model})
resource_obj = model.Resource.get(resource_view['resource_id'])
resource = md.resource_dictize(resource_obj, {'model': model})
organizationpage_views.append({
'resource_view': resource_view,
'resource': resource,
'package': md.package_dictize(resource_obj.package, {'model':model})
})
except Exception, ex:
print '\nDEBUG: '+str(ex)
def resource_update(context, data_dict):
model = context['model']
session = context['session']
user = context['user']
id = data_dict["id"]
schema = context.get('schema') or default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
raise NotFound(_('Resource was not found.'))
check_access('resource_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, resource_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
resource = resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return resource_dictize(resource, context)
def create_identify_resource_task(resource):
"""
Creates the celery task to identify the resource
:param resource: the resource to be identified
"""
task_id = make_uuid()
# We are using resource_dictize() just to force CKAN to provide an absolute url
# Note Maybe a more clean way to achive this would be to call something like
# url_for(controller='package', action='resource_download', id=package_id, resource_id=resource_id)
package_id = resource.as_dict()['package_id']
resource_dict = resource_dictize(resource, {'model': model})
resource_dict['package_id'] = package_id
context = _make_default_context()
context['resource_dict'] = resource_dict
celery.send_task('rasterstorer.identify', args=[context], task_id=task_id)
res_identify = model.Session.query(ResourceIngest).filter(
ResourceIngest.resource_id == resource.id).first()
if res_identify:
# This is when a user had previously rejected the ingestion workflow,
# but now wants to re-identify the resource
model.Session.delete(res_identify)
new_res_identify = ResourceIngest(task_id, resource.id,
ResourceStorerType.RASTER)
model.Session.add(new_res_identify)
model.Session.commit()
else:
# A newly created/updated resource needs to be identified
new_res_identify = ResourceIngest(task_id, resource.id,
ResourceStorerType.RASTER)
model.Session.add(new_res_identify)
def test_01_dictize_main_objects_simple(self):
context = {"model": model, "session": model.Session}
## package
pkg = model.Session.query(model.Package).filter_by(name="annakarenina").first()
result = table_dictize(pkg, context)
self.remove_changable_columns(result)
assert result == {
"author": None,
"author_email": None,
"license_id": u"other-open",
"maintainer": None,
"maintainer_email": None,
"name": u"annakarenina",
"notes": u"Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n",
"state": u"active",
"title": u"A Novel By Tolstoy",
"url": u"http://www.annakarenina.com",
"version": u"0.7a",
}, pprint(result)
## resource
resource = pkg.resource_groups[0].resources[0]
result = resource_dictize(resource, context)
self.remove_changable_columns(result)
assert result == {
u"alt_url": u"alt123",
"cache_last_updated": None,
"cache_url": None,
"description": u'Full text. Needs escaping: " Umlaut: \xfc',
"format": u"plain text",
"hash": u"abc123",
"last_modified": None,
"mimetype": None,
"mimetype_inner": None,
"name": None,
"position": 0,
"resource_type": None,
"size": None,
u"size_extra": u"123",
"state": u"active",
"url": u"http://www.annakarenina.com/download/x=1&y=2",
"webstore_last_updated": None,
"webstore_url": None,
}, pprint(result)
## package extra
key, package_extras = pkg._extras.popitem()
result = table_dictize(package_extras, context)
self.remove_changable_columns(result)
assert result == {"key": u"genre", "state": u"active", "value": u"romantic novel"}, pprint(result)
def _create_archiver_task(self, resource):
from ckan.lib.base import c
site_user = get_action('get_site_user')({
'model': model,
'ignore_auth': True,
'defer_commit': True
}, {})
# If the code that triggers this is run from the command line, the c
# stacked object proxy variable will not have been set up by the paste
# registry so will give an error saying no object has been registered
# for this thread. The easiest thing to do is to catch this, but it
# would be nice to have a config option so that the behaviour can be
# specified.
try:
c.user
except TypeError:
# This is no different from running the archiver from the command line:
# See https://github.com/okfn/ckanext-archiver/blob/master/ckanext/archiver/commands.py
username = site_user['name']
userapikey = site_user['apikey']
else:
user = model.User.by_name(c.user)
username = user.name
userapikey = user.apikey
context = json.dumps({
'site_url': self.site_url,
'apikey': userapikey,
'username': username,
'cache_url_root': self.cache_url_root,
'site_user_apikey': site_user['apikey']
})
res_dict = resource_dictize(resource, {'model': model})
data = json.dumps(res_dict)
task_id = make_uuid()
archiver_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'archiver',
'key': u'celery_task_id',
'value': task_id,
'error': u'',
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'user': site_user['name'],
'ignore_auth': True
}
get_action('task_status_update')(archiver_task_context,
archiver_task_status)
celery.send_task("archiver.update",
args=[context, data],
task_id=task_id)
log.debug(
'Archival of resource put into celery queue: %s url=%r user=%s site_user=%s site_url=%s',
resource.id, res_dict.get('url'), username, site_user['name'],
self.site_url)
def test_26_resource_show(self):
pkg = model.Package.get('annakarenina')
resource = pkg.resources[0]
postparams = '%s=1' % json.dumps({'id': resource.id})
res = self.app.post('/api/action/resource_show', params=postparams)
result = json.loads(res.body)['result']
resource_dict = resource_dictize(resource, {'model': model})
assert result == resource_dict, (result, resource_dict)
def test_26_resource_show(self):
pkg = model.Package.get('annakarenina')
resource = pkg.resources[0]
postparams = '%s=1' % json.dumps({'id': resource.id})
res = self.app.post('/api/action/resource_show', params=postparams)
result = json.loads(res.body)['result']
resource_dict = resource_dictize(resource, {'model': model})
result.pop('revision_timestamp')
assert result == resource_dict, (result, resource_dict)
def identify_resource(resource_obj):
user_api_key = _get_site_user()['apikey']
res_dict = resource_dictize(resource_obj, {'model': model})
resource = resource_obj.as_dict()
'''With resource_dictize we get the correct resource url even if dataset is in draft state '''
resource['url'] = res_dict['url']
data = json.dumps(resource)
job = jobs.enqueue(tasks.identify_resource, [data, user_api_key])
def test_01_dictize_main_objects_simple(self):
context = {"model": model, "session": model.Session}
## package
pkg = model.Session.query(
model.Package).filter_by(name='annakarenina').first()
result = table_dictize(pkg, context)
self.remove_changable_columns(result)
assert result == {
'author': None,
'author_email': None,
'license_id': u'other-open',
'maintainer': None,
'maintainer_email': None,
'name': u'annakarenina',
'notes':
u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
'state': u'active',
'title': u'A Novel By Tolstoy',
'url': u'http://www.annakarenina.com',
'version': u'0.7a'
}, pprint(result)
## resource
resource = pkg.resource_groups[0].resources[0]
result = resource_dictize(resource, context)
self.remove_changable_columns(result)
assert result == {
'alt_url': u'alt123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'size': u'123',
'state': u'active',
'url': u'http://www.annakarenina.com/download/x=1&y=2'
}, pprint(result)
## package extra
key, package_extras = pkg._extras.popitem()
result = table_dictize(package_extras, context)
self.remove_changable_columns(result)
assert result == {
'key': u'genre',
'state': u'active',
'value': u'romantic novel'
}, pprint(result)
def test_01_dictize_main_objects_simple(self):
context = {"model": model,
"session": model.Session}
## package
pkg = model.Session.query(model.Package).filter_by(name='annakarenina').first()
result = table_dictize(pkg, context)
self.remove_changable_columns(result)
assert result == {
'author': None,
'author_email': None,
'license_id': u'other-open',
'maintainer': None,
'maintainer_email': None,
'name': u'annakarenina',
'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
'state': u'active',
'title': u'A Novel By Tolstoy',
'url': u'http://www.annakarenina.com',
'version': u'0.7a'
}, pprint(result)
## resource
resource = pkg.resource_groups[0].resources[0]
result = resource_dictize(resource, context)
self.remove_changable_columns(result)
assert result == {
'alt_url': u'alt123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'size': u'123',
'state': u'active',
'url': u'http://www.annakarenina.com/download/x=1&y=2'}, pprint(result)
## package extra
key, package_extras = pkg._extras.popitem()
result = table_dictize(package_extras, context)
self.remove_changable_columns(result)
assert result == {
'key': u'genre',
'state': u'active',
'value': u'romantic novel'
}, pprint(result)
def test_hook(self):
testpackage = self.package
with self.app.flask_app.test_request_context():
resource_dict = model_dictize.resource_dictize(self.resource, {'model': model})
context = {
'model': model,
'session': model.Session,
'user': model.User.get('testsysadmin').name
}
# no preview for type "plain text"
preview_url = self.preview_url
result = self.app.get(preview_url, status=409)
assert 'No preview' in result.body, result.body
# no preview for type "ümlaut", should not fail
resource_dict['format'] = u'ümlaut'
l.action.update.resource_update(context, resource_dict)
result = self.app.get(preview_url, status=409)
assert 'No preview' in result.body, result.body
resource_dict['format'] = 'mock'
l.action.update.resource_update(context, resource_dict)
#there should be a preview for type "json"
preview_url = self.preview_url
result = self.app.get(preview_url, status=200)
assert 'mock-preview' in result.body
assert 'mock-preview.js' in result.body
assert self.plugin.calls['can_preview'] == 3, self.plugin.calls
assert self.plugin.calls['setup_template_variables'] == 1, self.plugin.calls
assert self.plugin.calls['preview_templates'] == 1, self.plugin.calls
# test whether the json preview is used
with self.app.flask_app.test_request_context():
preview_url = h.url_for(controller='package',
action='resource_datapreview',
id=testpackage.id,
resource_id=testpackage.resources[1].id)
result = self.app.get(preview_url, status=200)
assert 'mock-json-preview' in result.body
assert 'mock-json-preview.js' in result.body
assert self.plugin.calls['can_preview'] == 4, self.plugin.calls
assert self.plugin.calls['setup_template_variables'] == 1, self.plugin.calls
assert self.plugin.calls['preview_templates'] == 1, self.plugin.calls
def resource_show(context, data_dict):
model = context['model']
id = data_dict['id']
resource = model.Resource.get(id)
context['resource'] = resource
if not resource:
raise NotFound
check_access('resource_show', context, data_dict)
return model_dictize.resource_dictize(resource, context)
def resource_show(context, data_dict):
model = context['model']
id = data_dict['id']
resource = model.Resource.get(id)
context['resource'] = resource
if not resource:
raise NotFound
check_access('resource_show', context, data_dict)
return model_dictize.resource_dictize(resource, context)
def resource_update(context, data_dict):
'''Update a resource.
To update a resource you must be authorized to update the dataset that the
resource belongs to.
For further parameters see ``resource_create()``.
:param id: the id of the resource to update
:type id: string
:returns: the updated resource
:rtype: string
'''
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
if 'schema' in context:
schema = context['schema']
else:
package_plugin = lib_plugins.lookup_package_plugin(
resource.resource_group.package.type)
schema = package_plugin.update_package_schema()['resources']
data, errors = _validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def resource_update(context, data_dict):
'''Update a resource.
To update a resource you must be authorized to update the dataset that the
resource belongs to.
For further parameters see ``resource_create()``.
:param id: the id of the resource to update
:type id: string
:returns: the updated resource
:rtype: string
'''
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
if 'schema' in context:
schema = context['schema']
else:
package_plugin = lib_plugins.lookup_package_plugin(
resource.resource_group.package.type)
schema = package_plugin.update_package_schema()['resources']
data, errors = _validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def _create_archiver_task(self, resource):
from ckan.lib.base import c
site_user = get_action('get_site_user')(
{'model': model, 'ignore_auth': True, 'defer_commit': True}, {}
)
# If the code that triggers this is run from the command line, the c
# stacked object proxy variable will not have been set up by the paste
# registry so will give an error saying no object has been registered
# for this thread. The easiest thing to do is to catch this, but it
# would be nice to have a config option so that the behaviour can be
# specified.
try:
c.user
except TypeError:
# This is no different from running the archiver from the command line:
# See https://github.com/okfn/ckanext-archiver/blob/master/ckanext/archiver/commands.py
username = site_user['name']
userapikey = site_user['apikey']
else:
user = model.User.by_name(c.user)
username = user.name
userapikey = user.apikey
context = json.dumps({
'site_url': self.site_url,
'apikey': userapikey,
'username': username,
'cache_url_root': self.cache_url_root,
'site_user_apikey': site_user['apikey']
})
res_dict = resource_dictize(resource, {'model': model})
data = json.dumps(res_dict)
task_id = make_uuid()
archiver_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'archiver',
'key': u'celery_task_id',
'value': task_id,
'error': u'',
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'user': site_user['name'],
'ignore_auth': True
}
get_action('task_status_update')(archiver_task_context, archiver_task_status)
celery.send_task("archiver.update", args=[context, data], task_id=task_id)
log.debug('Archival of resource put into celery queue: %s url=%r user=%s site_user=%s site_url=%s', resource.id, res_dict.get('url'), username, site_user['name'], self.site_url)
def resource_show(context, data_dict):
model = context['model']
api = context.get('api_version') or '1'
id = data_dict['id']
resource = model.Resource.get(id)
context['resource'] = resource
if not resource:
raise NotFound
check_access('resource_show', context, data_dict)
return resource_dictize(resource, context)
def _get_canonical_view(package_id):
canonical = db.Civicdata_Featured.find(package_id=package_id, canonical=True).first()
if not canonical:
return None
resource_view = md.resource_view_dictize(
model.ResourceView.get(canonical.resource_view_id), {'model': model}
)
resource = md.resource_dictize(
model.Resource.get(resource_view['resource_id']), {'model': model}
)
return {'resource': resource, 'resource_view': resource_view}
def resource_show(context, data_dict):
model = context["model"]
api = context.get("api_version") or "1"
id = data_dict["id"]
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
raise NotFound
check_access("resource_show", context, data_dict)
return resource_dictize(resource, context)
def resource_show(context, data_dict):
model = context['model']
api = context.get('api_version') or '1'
id = data_dict['id']
resource = model.Resource.get(id)
context['resource'] = resource
if not resource:
raise NotFound
check_access('resource_show', context, data_dict)
return resource_dictize(resource, context)
def test_hook(self):
testpackage = self.package
resource_dict = model_dictize.resource_dictize(self.resource, {'model': model})
context = {
'model': model,
'session': model.Session,
'user': model.User.get('testsysadmin').name
}
# no preview for type "plain text"
preview_url = self.preview_url
result = self.app.get(preview_url, status=409)
assert 'No preview' in result.body, result.body
# no preview for type "ümlaut", should not fail
resource_dict['format'] = u'ümlaut'
l.action.update.resource_update(context, resource_dict)
result = self.app.get(preview_url, status=409)
assert 'No preview' in result.body, result.body
resource_dict['format'] = 'mock'
l.action.update.resource_update(context, resource_dict)
#there should be a preview for type "json"
preview_url = self.preview_url
result = self.app.get(preview_url, status=200)
assert 'mock-preview' in result.body
assert 'mock-preview.js' in result.body
assert self.plugin.calls['can_preview'] == 3, self.plugin.calls
assert self.plugin.calls['setup_template_variables'] == 1, self.plugin.calls
assert self.plugin.calls['preview_templates'] == 1, self.plugin.calls
# test whether the json preview is used
preview_url = h.url_for(controller='package',
action='resource_datapreview',
id=testpackage.id,
resource_id=testpackage.resources[1].id)
result = self.app.get(preview_url, status=200)
assert 'mock-json-preview' in result.body
assert 'mock-json-preview.js' in result.body
assert self.plugin.calls['can_preview'] == 4, self.plugin.calls
assert self.plugin.calls['setup_template_variables'] == 1, self.plugin.calls
assert self.plugin.calls['preview_templates'] == 1, self.plugin.calls
def update_vector_storer_task(resource):
user = _get_site_user()
resource_package_id = resource.as_dict()['package_id']
resource_list_to_delete = _get_child_resources(resource.as_dict())
context = json.dumps({'resource_list_to_delete': resource_list_to_delete,
'package_id': resource_package_id,
'site_url': _get_site_url(),
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'user': user.get('name'),
'db_params': config['ckan.datastore.write_url']})
geoserver_context = _get_geoserver_context()
data = json.dumps(resource_dictize(resource, {'model': model}))
task_id = make_uuid()
celery.send_task('vectorstorer.update', args=[geoserver_context, context, data], task_id=task_id)
def identify_resource(resource_obj):
user_api_key = _get_site_user()['apikey']
res_dict = resource_dictize(resource_obj, {'model': model})
resource=resource_obj.as_dict()
'''With resource_dictize we get the correct resource url even if dataset is in draft state '''
resource['url']=res_dict['url']
task_id = make_uuid()
data = json.dumps(resource)
celery.send_task('vectorstorer.identify_resource', args=[data,user_api_key], task_id=task_id)
res_identify = ResourceIdentify(task_id,resource['id'])
ckan.model.Session.add(res_identify)
def update_ingest_resource(resource):
package_id = resource.as_dict()['package_id']
resource_list_to_delete = _get_child_resources(resource.as_dict())
context = _make_default_context()
context.update({
'resource_list_to_delete': resource_list_to_delete,
'package_id': package_id,
'db_params': config['ckan.datastore.write_url'],
})
geoserver_context = _make_geoserver_context()
resource_dict = resource_dictize(resource, {'model': model})
task_id = make_uuid()
celery.send_task('vectorstorer.update',
args=[resource_dict, context, geoserver_context],
task_id=task_id)
def resource_update(context, data_dict):
"""Update a resource.
To update a resource you must be authorized to update the dataset that the
resource belongs to.
For further parameters see ``resource_create()``.
:param id: the id of the resource to update
:type id: string
:returns: the updated resource
:rtype: string
"""
model = context["model"]
user = context["user"]
id = _get_or_bust(data_dict, "id")
schema = context.get("schema") or ckan.logic.schema.default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error("Could not find resource " + id)
raise NotFound(_("Resource was not found."))
_check_access("resource_update", context, data_dict)
data, errors = _validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if "message" in context:
rev.message = context["message"]
else:
rev.message = _(u"REST API: Update object %s") % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get("defer_commit"):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def update_vector_storer_task(resource):
user = _get_site_user()
resource_package_id = resource.as_dict()['package_id']
resource_list_to_delete = _get_child_resources(resource.as_dict())
context = json.dumps({
'resource_list_to_delete': resource_list_to_delete,
'package_id': resource_package_id,
'site_url': _get_site_url(),
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'user': user.get('name'),
'db_params': config['ckan.datastore.write_url']
})
geoserver_context = _get_geoserver_context()
data = json.dumps(resource_dictize(resource, {'model': model}))
log.debug('update vectorstore task')
jobs.enqueue(tasks.vectorstorer_update, [geoserver_context, context, data])
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.Session.commit()
new_resource = {
"mimetype": None,
u"alt_url": u"empty resource group id",
"hash": u"abc123",
"description": u'Full text. Needs escaping: " Umlaut: \xfc',
"format": u"plain text",
"url": u"http://test_new",
"cache_url": None,
"cache_last_updated": None,
"state": u"active",
"mimetype_inner": None,
"url_type": None,
"last_modified": None,
"position": 0,
"size": None,
"size_extra": u"123",
"resource_type": None,
"name": None,
"package_id": "", # Just so we can save
}
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource["package_id"]
res = (
model.Session.query(model.Resource)
.filter_by(url=u"http://test_new")
.one()
)
res_dictized = self.remove_changable_columns(
resource_dictize(res, context), True
)
assert res_dictized == new_resource, res_dictized
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'mimetype': None,
u'alt_url': u'empty resource group id',
'hash': u'abc123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'url': u'http://test_new',
'cache_url': None,
'webstore_url': None,
'cache_last_updated': None,
'state': u'active',
'mimetype_inner': None,
'webstore_last_updated': None,
'url_type': None,
'last_modified': None,
'position': 0,
'size': None,
'size_extra': u'123',
'resource_type': None,
'name': None,
'package_id': '' # Just so we can save
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource['package_id']
res = model.Session.query(
model.Resource).filter_by(url=u'http://test_new').one()
res_dictized = self.remove_changable_columns(
resource_dictize(res, context), True)
assert res_dictized == new_resource, res_dictized
def _get_canonical_view(package_id):
canonical = db.Featured.find(package_id=package_id, canonical=True).first()
if not canonical:
return None
resource_view = model.ResourceView.get(canonical.resource_view_id)
if resource_view is None:
return None
resource_view_dictized = md.resource_view_dictize(resource_view,
{'model': model})
resource = md.resource_dictize(
model.Resource.get(resource_view_dictized['resource_id']),
{'model': model})
return {'resource': resource, 'resource_view': resource_view_dictized}
def create_vector_storer_task(resource, extra_params = None):
user = _get_site_user()
resource_package_id = resource.as_dict()['package_id']
cont = {'package_id': resource_package_id,
'site_url': _get_site_url(),
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'user': user.get('name'),
'db_params': config['ckan.datastore.write_url']}
if extra_params:
for key, value in extra_params.iteritems():
cont[key] = value
context = json.dumps(cont)
geoserver_context = _get_geoserver_context()
data = json.dumps(resource_dictize(resource, {'model': model}))
task_id = make_uuid()
celery.send_task('vectorstorer.upload', args=[geoserver_context, context, data], task_id=task_id)
def test_14_resource_no_id(self):
context = {"model": model,
"session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'mimetype': None,
u'alt_url': u'empty resource group id',
'hash': u'abc123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'url': u'http://test_new',
'cache_url': None,
'webstore_url': None,
'cache_last_updated': None,
'state': u'active',
'mimetype_inner': None,
'webstore_last_updated': None,
'url_type': None,
'last_modified': None,
'position': 0,
'size': None,
'size_extra': u'123',
'resource_type': None,
'name': None,
'package_id':'' # Just so we can save
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource['package_id']
res = model.Session.query(model.Resource).filter_by(url=u'http://test_new').one()
res_dictized = self.remove_changable_columns(resource_dictize(res, context), True)
assert res_dictized == new_resource, res_dictized
def create_identify_resource_task(resource):
"""
Creates the celery task to identify the resource
:param resource: the resource to be identified
"""
task_id = make_uuid()
# We are using resource_dictize() just to force CKAN to provide an absolute url
# Note Maybe a more clean way to achive this would be to call something like
# url_for(controller='package', action='resource_download', id=package_id, resource_id=resource_id)
package_id = resource.as_dict()['package_id']
resource_dict = resource_dictize(resource, {'model': model})
resource_dict['package_id'] = package_id
context = _make_default_context()
context['resource_dict'] = resource_dict
celery.send_task(
'rasterstorer.identify',
args=[context],
task_id=task_id
)
res_identify = model.Session.query(ResourceIngest).filter(
ResourceIngest.resource_id == resource.id).first()
if res_identify:
# This is when a user had previously rejected the ingestion workflow,
# but now wants to re-identify the resource
model.Session.delete(res_identify)
new_res_identify = ResourceIngest(
task_id,
resource.id,
ResourceStorerType.RASTER
)
model.Session.add(new_res_identify)
model.Session.commit()
else:
# A newly created/updated resource needs to be identified
new_res_identify = ResourceIngest(
task_id,
resource.id,
ResourceStorerType.RASTER
)
model.Session.add(new_res_identify)
def resource_update(context, data_dict):
model = context["model"]
session = context["session"]
user = context["user"]
id = data_dict["id"]
schema = context.get("schema") or default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
raise NotFound(_("Resource was not found."))
# check authentication against the resource package
# TODO: can check_access be used against a resource?
query = (
session.query(model.Package)
.join(model.ResourceGroup)
.join(model.Resource)
.filter(model.ResourceGroup.id == resource.resource_group_id)
)
pkg = query.first()
if not pkg:
raise NotFound(_("No package found for this resource, cannot check auth."))
check_access("package_update", context, package_dictize(pkg, context))
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, resource_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if "message" in context:
rev.message = context["message"]
else:
rev.message = _(u"REST API: Update object %s") % data.get("name")
resource = resource_dict_save(data, context)
model.repo.commit()
return resource_dictize(resource, context)
def create_vector_storer_task(resource, extra_params=None):
user = _get_site_user()
resource_package_id = resource.as_dict()['package_id']
cont = {
'package_id': resource_package_id,
'site_url': _get_site_url(),
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'user': user.get('name'),
'db_params': config['ckan.datastore.write_url']
}
if extra_params:
for key, value in extra_params.iteritems():
cont[key] = value
context = json.dumps(cont)
geoserver_context = _get_geoserver_context()
data = json.dumps(resource_dictize(resource, {'model': model}))
log.debug('create vectorstore task')
jobs.enqueue(tasks.vectorstorer_upload, [geoserver_context, context, data])
def create_ingest_resource(resource, layer_params):
package_id = resource.as_dict()['package_id']
context = _make_default_context()
context.update({
'package_id': package_id,
'db_params': config['ckan.datastore.write_url'],
'layer_params': layer_params
})
geoserver_context = _make_geoserver_context()
resource_dict = resource_dictize(resource, {'model': model})
task_id = make_uuid()
celery.send_task('vectorstorer.upload',
args=[resource_dict, context, geoserver_context],
task_id=task_id)
res_ingest = model.Session.query(ResourceIngest).filter(
ResourceIngest.resource_id == resource.id).first()
res_ingest.status = IngestStatus.PUBLISHED
res_ingest.celery_task_id = task_id
model.Session.commit()
def _create_datastorer_task(self, resource):
user = get_action('get_site_user')({
'model': model,
'ignore_auth': True,
'defer_commit': True
}, {})
context = {
'site_url': self._get_site_url(),
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'username': user.get('name'),
}
if self.sample_size:
context['sample_size'] = self.sample_size
data = resource_dictize(resource, {'model': model})
task_id = make_uuid()
datastorer_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'datastorer',
'key': u'celery_task_id',
'value': task_id,
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'user': user.get('name'),
}
get_action('task_status_update')(archiver_task_context,
datastorer_task_status)
celery.send_task("datastorer.upload",
args=[json.dumps(context),
json.dumps(data)],
countdown=15,
task_id=task_id)
logger.info('Sent task: datastorer.upload id=%s context=%r' %
(task_id, context))
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
"mimetype": None,
u"alt_url": u"empty resource group id",
"hash": u"abc123",
"description": u'Full text. Needs escaping: " Umlaut: \xfc',
"format": u"plain text",
"url": u"http://test_new",
"cache_url": None,
"cache_last_updated": None,
"state": u"active",
"mimetype_inner": None,
"url_type": None,
"last_modified": None,
"position": 0,
"size": None,
"size_extra": u"123",
"resource_type": None,
"name": None,
"package_id": "", # Just so we can save
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource["package_id"]
res = model.Session.query(model.Resource).filter_by(url=u"http://test_new").one()
res_dictized = self.remove_changable_columns(resource_dictize(res, context), True)
assert res_dictized == new_resource, res_dictized
def delete_ingest_resource(resource, pkg_delete=False):
resource_dict = resource_dictize(resource, {'model': model})
resource_list_to_delete = None
if ((resource_dict['format'] == WMSResource.FORMAT
or resource_dict['format'] == DBTableResource.FORMAT)
and 'vectorstorer_resource' in resource_dict):
if pkg_delete:
resource_list_to_delete = _get_child_resources(resource)
else:
resource_list_to_delete = _get_child_resources(resource)
context = _make_default_context()
context.update({
'resource_list_to_delete': resource_list_to_delete,
'db_params': config['ckan.datastore.write_url']
})
geoserver_context = _make_geoserver_context()
task_id = make_uuid()
celery.send_task('vectorstorer.delete',
args=[resource_dict, context, geoserver_context],
task_id=task_id)
if 'vectorstorer_resource' in resource and not pkg_delete:
_delete_child_resources(resource)
def broken_resource_links_by_dataset():
"""
Return a list of named tuples, one for each dataset that contains
broken resource links (defined as resources with an openness score of 0).
The named tuple is of the form:
(name (str), title (str), resources (list of dicts))
"""
query = Session.query(Package.name, Package.title, Resource)\
.join(ResourceGroup, Package.id==ResourceGroup.package_id)\
.join(Resource)\
.join(TaskStatus, TaskStatus.entity_id==Resource.id)\
.filter(TaskStatus.key==u'openness_score')\
.filter(TaskStatus.value==u'0')\
.distinct()
context = {'model': model, 'session': model.Session}
results = {}
for name, title, resource in query:
resource = resource_dictize(resource, context)
data = {
'entity_id': resource['id'],
'task_type': 'qa',
'key': 'openness_score_reason'
}
status = get_action('task_status_show')(context, data)
resource['openness_score_reason'] = status.get('value')
if name in results:
results[name].resources.append(resource)
else:
DatasetTuple = namedtuple('DatasetTuple',
['name', 'title', 'resources'])
results[name] = DatasetTuple(name, title or name, [resource])
return results.values()
def _create_datastorer_task(self, resource):
user = get_action('get_site_user')({'model': model,
'ignore_auth': True,
'defer_commit': True}, {})
context = {
'site_url': self._get_site_url(),
'apikey': user.get('apikey'),
'site_user_apikey': user.get('apikey'),
'username': user.get('name'),
}
if self.sample_size:
context['sample_size'] = self.sample_size
data = resource_dictize(resource, {'model': model})
task_id = make_uuid()
datastorer_task_status = {
'entity_id': resource.id,
'entity_type': u'resource',
'task_type': u'datastorer',
'key': u'celery_task_id',
'value': task_id,
'last_updated': datetime.now().isoformat()
}
archiver_task_context = {
'model': model,
'user': user.get('name'),
}
get_action('task_status_update')(archiver_task_context,
datastorer_task_status)
celery.send_task("datastorer.upload",
args=[json.dumps(context), json.dumps(data)],
countdown=15,
task_id=task_id)
logger.info('Sent task: datastorer.upload id=%s context=%r' %(task_id, context))
def test_01_package_list(self, app):
res = json.loads(
app.post(
"/api/action/package_list",
headers={"content-type": "application/json"},
).body
)
assert res["success"] is True
assert len(res["result"]) == 2
assert "warandpeace" in res["result"]
assert "annakarenina" in res["result"]
assert "/api/3/action/help_show?name=package_list" in res["help"]
res = json.loads(
app.post("/api/action/package_list", json={"limit": 1}).body
)
assert res["success"] is True
assert len(res["result"]) == 1
assert (
"warandpeace" in res["result"] or "annakarenina" in res["result"]
)
# Test GET request
res = json.loads(app.get("/api/action/package_list").body)
assert len(res["result"]) == 2
assert "warandpeace" in res["result"]
assert "annakarenina" in res["result"]
# def test_01_package_list_private(self):
tests.call_action_api(
app,
"organization_create",
name="test_org_2",
apikey=self.sysadmin_user.apikey,
)
tests.call_action_api(
app,
"package_create",
name="public_dataset",
owner_org="test_org_2",
apikey=self.sysadmin_user.apikey,
)
res = tests.call_action_api(app, "package_list")
assert len(res) == 3
assert "warandpeace" in res
assert "annakarenina" in res
assert "public_dataset" in res
tests.call_action_api(
app,
"package_create",
name="private_dataset",
owner_org="test_org_2",
private=True,
apikey=self.sysadmin_user.apikey,
)
res = tests.call_action_api(app, "package_list")
assert len(res) == 3
assert "warandpeace" in res
assert "annakarenina" in res
assert "public_dataset" in res
assert "private_dataset" not in res
# def test_02_package_autocomplete_match_name(self):
res = app.post("/api/action/package_autocomplete", json={"q": "war", "limit": 5})
res_obj = json.loads(res.body)
assert res_obj["success"]
assert res_obj["result"][0]["name"] == "warandpeace"
assert res_obj["result"][0]["title"] == "A Wonderful Story"
assert res_obj["result"][0]["match_field"] == "name"
assert res_obj["result"][0]["match_displayed"] == "warandpeace"
# def test_02_package_autocomplete_match_title(self):
res = app.post("/api/action/package_autocomplete", json={"q": "won", "limit": 5})
res_obj = json.loads(res.body)
assert res_obj["success"]
assert res_obj["result"][0]["name"] == "warandpeace"
assert res_obj["result"][0]["title"] == "A Wonderful Story"
assert res_obj["result"][0]["match_field"] == "title"
assert (
res_obj["result"][0]["match_displayed"]
== "A Wonderful Story (warandpeace)"
)
# def test_03_create_private_package(self):
# Make an organization, because private datasets must belong to one.
organization = tests.call_action_api(
app,
"organization_create",
name="test_org",
apikey=self.sysadmin_user.apikey,
)
# Create a dataset without specifying visibility
package_dict = {
"extras": [{"key": u"original media", "value": u'"book"'}],
"license_id": u"other-open",
"maintainer_email": None,
"name": u"annakarenina_vis",
"notes": u"Some test now",
"resources": [
{
"alt_url": u"alt123",
"description": u"Full text.",
"extras": {u"alt_url": u"alt123", u"size": u"123"},
"format": u"plain text",
"hash": u"abc123",
"position": 0,
"url": u"http://datahub.io/download/",
},
{
"alt_url": u"alt345",
"description": u"Index of the novel",
"extras": {u"alt_url": u"alt345", u"size": u"345"},
"format": u"JSON",
"hash": u"def456",
"position": 1,
"url": u"http://datahub.io/index.json",
},
],
"tags": [{"name": u"russian"}, {"name": u"tolstoy"}],
"title": u"A Novel By Tolstoy",
"url": u"http://datahub.io",
"owner_org": organization["id"],
"version": u"0.7a",
}
package_created = tests.call_action_api(
app,
"package_create",
apikey=self.sysadmin_user.apikey,
**package_dict
)
assert package_created["private"] is False
# Create a new one, explicitly saying it is public
package_dict["name"] = u"annakareninanew_vis_public"
package_dict["private"] = False
package_created_public = tests.call_action_api(
app,
"package_create",
apikey=self.sysadmin_user.apikey,
**package_dict
)
assert package_created_public["private"] is False
# Create a new one, explicitly saying it is private
package_dict["name"] = u"annakareninanew_vis_private"
package_dict["private"] = True
package_created_private = tests.call_action_api(
app,
"package_create",
apikey=self.sysadmin_user.apikey,
**package_dict
)
assert package_created_private["private"] is True
# def test_41_create_resource(self):
anna_id = model.Package.by_name(u"annakarenina").id
resource = {"package_id": anna_id, "url": "http://new_url"}
api_key = six.ensure_text(model.User.get("testsysadmin").apikey)
res = app.post(
"/api/action/resource_create",
json=resource,
extra_environ={"Authorization": str(api_key)},
)
resource = json.loads(res.body)["result"]
assert resource["url"] == "http://new_url"
# def test_42_create_resource_with_error(self):
anna_id = model.Package.by_name(u"annakarenina").id
resource = {
"package_id": anna_id,
"url": "new_url",
"created": "bad_date",
}
api_key = six.ensure_text(model.User.get("testsysadmin").apikey)
res = app.post(
"/api/action/resource_create",
json=resource,
extra_environ={"Authorization": str(api_key)},
status=StatusCodes.STATUS_409_CONFLICT,
)
assert json.loads(res.body)["error"] == {
"__type": "Validation Error",
"created": ["Date format incorrect"],
}
# def test_10_user_create_parameters_missing(self):
user_dict = {}
res = app.post(
"/api/action/user_create",
json=user_dict,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
status=StatusCodes.STATUS_409_CONFLICT,
)
res_obj = json.loads(res.body)
assert res_obj["error"] == {
"__type": "Validation Error",
"name": ["Missing value"],
"email": ["Missing value"],
"password": ["Missing value"],
}
assert "/api/3/action/help_show?name=user_create" in res_obj["help"]
assert res_obj["success"] is False
# def test_11_user_create_wrong_password(self):
user_dict = {
"name": "test_create_from_action_api_2",
"email": "*****@*****.**",
"password": "******",
} # Too short
res = app.post(
"/api/action/user_create",
json=user_dict,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
status=StatusCodes.STATUS_409_CONFLICT,
)
res_obj = json.loads(res.body)
assert "/api/3/action/help_show?name=user_create" in res_obj["help"]
assert res_obj["success"] is False
assert res_obj["error"] == {
"__type": "Validation Error",
"password": ["Your password must be 8 characters or longer"],
}
# def test_12_user_update(self):
normal_user_dict = {
"id": self.normal_user.id,
"name": self.normal_user.name,
"fullname": "Updated normal user full name",
"email": "*****@*****.**",
"about": "Updated normal user about",
}
sysadmin_user_dict = {
"id": self.sysadmin_user.id,
"fullname": "Updated sysadmin user full name",
"email": "*****@*****.**",
"about": "Updated sysadmin user about",
}
# Normal users can update themselves
res = app.post(
"/api/action/user_update",
json=normal_user_dict,
extra_environ={"Authorization": str(self.normal_user.apikey)},
)
res_obj = json.loads(res.body)
assert "/api/3/action/help_show?name=user_update" in res_obj["help"]
assert res_obj["success"] == True
result = res_obj["result"]
assert result["id"] == self.normal_user.id
assert result["name"] == self.normal_user.name
assert result["fullname"] == normal_user_dict["fullname"]
assert result["about"] == normal_user_dict["about"]
assert "apikey" in result
assert "created" in result
assert "display_name" in result
assert "number_created_packages" in result
assert not "password" in result
# Sysadmin users can update themselves
res = app.post(
"/api/action/user_update",
json=sysadmin_user_dict,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
res_obj = json.loads(res.body)
assert "/api/3/action/help_show?name=user_update" in res_obj["help"]
assert res_obj["success"] == True
result = res_obj["result"]
assert result["id"] == self.sysadmin_user.id
assert result["name"] == self.sysadmin_user.name
assert result["fullname"] == sysadmin_user_dict["fullname"]
assert result["about"] == sysadmin_user_dict["about"]
# Sysadmin users can update all users
res = app.post(
"/api/action/user_update",
json=normal_user_dict,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
res_obj = json.loads(res.body)
assert "/api/3/action/help_show?name=user_update" in res_obj["help"]
assert res_obj["success"] == True
result = res_obj["result"]
assert result["id"] == self.normal_user.id
assert result["name"] == self.normal_user.name
assert result["fullname"] == normal_user_dict["fullname"]
assert result["about"] == normal_user_dict["about"]
# Normal users can not update other users
res = app.post(
"/api/action/user_update",
json=sysadmin_user_dict,
extra_environ={"Authorization": str(self.normal_user.apikey)},
status=StatusCodes.STATUS_403_ACCESS_DENIED,
)
res_obj = json.loads(res.body)
assert "/api/3/action/help_show?name=user_update" in res_obj["help"]
assert res_obj["error"]["__type"] == "Authorization Error"
assert res_obj["success"] is False
# def test_12_user_update_errors(self):
test_calls = (
# Empty name
{
"user_dict": {
"id": self.normal_user.id,
"name": "",
"email": "*****@*****.**",
},
"messages": [("name", "Must be at least 2 characters long")],
},
# Invalid characters in name
{
"user_dict": {
"id": self.normal_user.id,
"name": "i++%",
"email": "*****@*****.**",
},
"messages": [
("name", "Must be purely lowercase alphanumeric")
],
},
# Existing name
{
"user_dict": {
"id": self.normal_user.id,
"name": self.sysadmin_user.name,
"email": "*****@*****.**",
},
"messages": [("name", "That login name is not available")],
},
# Missing email
{
"user_dict": {
"id": self.normal_user.id,
"name": self.normal_user.name,
},
"messages": [("email", "Missing value")],
},
)
for test_call in test_calls:
res = app.post(
"/api/action/user_update",
json=test_call["user_dict"],
extra_environ={"Authorization": str(self.normal_user.apikey)},
status=StatusCodes.STATUS_409_CONFLICT,
)
res_obj = json.loads(res.body)
for expected_message in test_call["messages"]:
assert expected_message[1] in "".join(
res_obj["error"][expected_message[0]]
)
# def test_user_delete(self):
name = "normal_user"
CreateTestData.create_user(name)
user = model.User.get(name)
user_dict = {"id": user.id}
res = app.post(
"/api/action/user_delete",
json=user_dict,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
res_obj = json.loads(res.body)
deleted_user = model.User.get(name)
assert res_obj["success"] is True
assert deleted_user.is_deleted(), deleted_user
# def test_user_delete_requires_data_dict_with_key_id(self):
user_dict = {"name": "normal_user"}
res = app.post(
"/api/action/user_delete",
json=user_dict,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
status=StatusCodes.STATUS_409_CONFLICT,
)
res_obj = json.loads(res.body)
assert res_obj["success"] is False
assert res_obj["error"]["id"] == ["Missing value"]
# def test_16_user_autocomplete(self):
# Create deleted user to make sure he won't appear in the user_list
deleted_user = CreateTestData.create_user("joe")
deleted_user.delete()
model.repo.commit()
# Empty query
res = app.post(
"/api/action/user_autocomplete",
json={},
status=StatusCodes.STATUS_409_CONFLICT,
)
res_obj = json.loads(res.body)
assert (
"/api/3/action/help_show?name=user_autocomplete" in res_obj["help"]
)
assert res_obj["success"] is False
# Normal query
res = app.post("/api/action/user_autocomplete", json={"q": "joe"})
res_obj = json.loads(res.body)
assert res_obj["result"][0]["name"] == "joeadmin"
assert "id", "fullname" in res_obj["result"][0]
# def test_17_bad_action(self):
# Empty query
res = app.post(
"/api/action/bad_action_name", json={}, status=400
)
res_obj = json.loads(res.body)
assert (
res_obj == u"Bad request - Action name not known: bad_action_name"
)
# def test_20_task_status_update(self):
package_created = _add_basic_package(app, u"test_task_status_update")
task_status = {
"entity_id": package_created["id"],
"entity_type": u"package",
"task_type": u"test_task",
"key": u"test_key",
"value": u"test_value",
"state": u"test_state",
"error": u"test_error",
}
res = app.post(
"/api/action/task_status_update",
json=task_status,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_status_updated = json.loads(res.body)["result"]
task_status_id = task_status_updated.pop("id")
task_status_updated.pop("last_updated")
assert task_status_updated == task_status
task_status_updated["id"] = task_status_id
task_status_updated["value"] = u"test_value_2"
res = app.post(
"/api/action/task_status_update",
json=task_status_updated,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_status_updated_2 = json.loads(res.body)["result"]
task_status_updated_2.pop("last_updated")
assert task_status_updated_2 == task_status_updated
# def test_21_task_status_update_many(self):
package_created = _add_basic_package(
app, u"test_task_status_update_many"
)
task_statuses = {
"data": [
{
"entity_id": package_created["id"],
"entity_type": u"package",
"task_type": u"test_task",
"key": u"test_task_1",
"value": u"test_value_1",
"state": u"test_state",
"error": u"test_error",
},
{
"entity_id": package_created["id"],
"entity_type": u"package",
"task_type": u"test_task",
"key": u"test_task_2",
"value": u"test_value_2",
"state": u"test_state",
"error": u"test_error",
},
]
}
res = app.post(
"/api/action/task_status_update_many",
json=task_statuses,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_statuses_updated = json.loads(res.body)["result"]["results"]
for i in range(len(task_statuses["data"])):
task_status = task_statuses["data"][i]
task_status_updated = task_statuses_updated[i]
task_status_updated.pop("id")
task_status_updated.pop("last_updated")
assert task_status == task_status_updated, (
task_status_updated,
task_status,
i,
)
# def test_22_task_status_normal_user_not_authorized(self):
task_status = {}
res = app.post(
"/api/action/task_status_update",
json=task_status,
extra_environ={"Authorization": str(self.normal_user.apikey)},
status=StatusCodes.STATUS_403_ACCESS_DENIED,
)
res_obj = json.loads(res.body)
assert (
"/api/3/action/help_show?name=task_status_update"
in res_obj["help"]
)
assert res_obj["success"] is False
assert res_obj["error"]["__type"] == "Authorization Error"
# def test_23_task_status_validation(self):
task_status = {}
res = app.post(
"/api/action/task_status_update",
json=task_status,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
status=StatusCodes.STATUS_409_CONFLICT,
)
# def test_24_task_status_show(self):
package_created = _add_basic_package(app, u"test_task_status_show")
task_status = {
"entity_id": package_created["id"],
"entity_type": u"package",
"task_type": u"test_task",
"key": u"test_task_status_show",
"value": u"test_value",
"state": u"test_state",
"error": u"test_error",
}
res = app.post(
"/api/action/task_status_update",
json=task_status,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_status_updated = json.loads(res.body)["result"]
# make sure show works when giving a task status ID
res = app.post(
"/api/action/task_status_show",
json={"id": task_status_updated["id"]},
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_status_show = res.json["result"]
task_status_show.pop("last_updated")
task_status_updated.pop("last_updated")
assert task_status_show == task_status_updated, (
task_status_show,
task_status_updated,
)
# make sure show works when giving a (entity_id, task_type, key) tuple
res = app.post(
"/api/action/task_status_show",
json={
"entity_id": task_status["entity_id"],
"task_type": task_status["task_type"],
"key": task_status["key"],
},
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_status_show = json.loads(res.body)["result"]
task_status_show.pop("last_updated")
assert task_status_show == task_status_updated, (
task_status_show,
task_status_updated,
)
# def test_25_task_status_delete(self):
package_created = _add_basic_package(app, u"test_task_status_delete")
task_status = {
"entity_id": package_created["id"],
"entity_type": u"package",
"task_type": u"test_task",
"key": u"test_task_status_delete",
"value": u"test_value",
"state": u"test_state",
"error": u"test_error",
}
res = app.post(
"/api/action/task_status_update",
json=task_status,
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_status_updated = json.loads(res.body)["result"]
res = app.post(
"/api/action/task_status_delete",
json={"id": task_status_updated["id"]},
extra_environ={"Authorization": str(self.sysadmin_user.apikey)},
)
task_status_delete = json.loads(res.body)
assert task_status_delete["success"] == True
# def test_26_resource_show(self):
pkg = model.Package.get("annakarenina")
resource = pkg.resources[0]
res = app.post("/api/action/resource_show", json={"id": resource.id})
result = json.loads(res.body)["result"]
resource_dict = resource_dictize(resource, {"model": model})
assert result == resource_dict, (result, resource_dict)
# def test_27_get_site_user_not_authorized(self):
with pytest.raises(NotAuthorized):
get_action("get_site_user")({"model": model, "user": ""}, {})
# user = model.User.get('test.ckan.net')
# assert not user
site_id = config.get("ckan.site_id")
user = get_action("get_site_user")(
{"model": model, "ignore_auth": True}, {}
)
assert user["name"] == site_id
user = model.User.get(site_id)
assert user
user = get_action("get_site_user")(
{"model": model, "ignore_auth": True}, {}
)
assert user["name"] == site_id
user = model.Session.query(model.User).filter_by(name=site_id).one()
assert user
# def test_28_group_package_show(self):
group_id = model.Group.get("david").id
group_packages = get_action("group_package_show")(
{
"model": model,
"user": self.normal_user.name,
"ignore_auth": True,
},
{"id": group_id},
)
assert len(group_packages) == 2, group_packages
group_names = set([g.get("name") for g in group_packages])
assert group_names == set(["annakarenina", "warandpeace"]), group_names
# def test_30_status_show(self):
res = app.post("/api/action/status_show", json={})
status = json.loads(res.body)["result"]
assert status["site_title"] == "CKAN"
assert status["ckan_version"] == ckan.__version__
assert status["site_url"] == "http://test.ckan.net"
# def test_31_bad_request_format(self):
res = app.post(
"/api/action/package_list", json=six.ensure_str("not a dict"), status=400
)
assert (
"Bad request - JSON Error: Request data JSON decoded to "
) in res
assert (
"'not a dict' but it needs to be a dictionary."
) in res
# def test_31_bad_request_format_not_json(self):
res = app.post(
"/api/action/package_list", data="=1", status=400, content_type="application/json"
)
assert body_contains(
res, "Bad request - JSON Error: Error decoding JSON data."
)
# def test_32_get_domain_object(self):
anna = model.Package.by_name(u"annakarenina")
assert get_domain_object(model, anna.name).name == anna.name
assert get_domain_object(model, anna.id).name == anna.name
group = model.Group.by_name(u"david")
assert get_domain_object(model, group.name).name == group.name
assert get_domain_object(model, group.id).name == group.name
# def test_41_missing_action(self):
try:
get_action("unicorns")
assert False, "We found a non-existent action"
except KeyError:
assert True
# def test_42_resource_search_with_single_field_query(self):
request_body = {"query": ["description:index"]}
response = app.post("/api/action/resource_search", json=request_body)
result = json.loads(response.body)["result"]["results"]
count = json.loads(response.body)["result"]["count"]
## Due to the side-effect of previously run tests, there may be extra
## resources in the results. So just check that each found Resource
## matches the search criteria
assert count > 0
for resource in result:
assert "index" in resource["description"].lower()
# def test_42_resource_search_across_multiple_fields(self):
request_body = {"query": ["description:index", "format:json"]}
response = app.post("/api/action/resource_search", json=request_body)
result = json.loads(response.body)["result"]["results"]
count = json.loads(response.body)["result"]["count"]
## Due to the side-effect of previously run tests, there may be extra
## resources in the results. So just check that each found Resource
## matches the search criteria
assert count > 0
for resource in result:
assert "index" in resource["description"].lower()
assert "json" in resource["format"].lower()
# def test_42_resource_search_test_percentage_is_escaped(self):
request_body = {"query": ["description:index%"]}
response = app.post("/api/action/resource_search", json=request_body)
count = json.loads(response.body)["result"]["count"]
# There shouldn't be any results. If the '%' character wasn't
# escaped correctly, then the search would match because of the
# unescaped wildcard.
assert count == 0
# def test_42_resource_search_fields_parameter_still_accepted(self):
"""The fields parameter is deprecated, but check it still works.
Remove this test when removing the fields parameter. (#2603)
"""
request_body = {"fields": {"description": "index"}}
response = app.post("/api/action/resource_search", json=request_body)
result = json.loads(response.body)["result"]["results"]
count = json.loads(response.body)["result"]["count"]
## Due to the side-effect of previously run tests, there may be extra
## resources in the results. So just check that each found Resource
## matches the search criteria
assert count > 0
for resource in result:
assert "index" in resource["description"].lower()
# def test_42_resource_search_accessible_via_get_request(self):
response = app.get(
"/api/action/resource_search"
"?query=description:index&query=format:json"
)
result = json.loads(response.body)["result"]["results"]
count = json.loads(response.body)["result"]["count"]
## Due to the side-effect of previously run tests, there may be extra
## resources in the results. So just check that each found Resource
## matches the search criteria
assert count > 0
for resource in result:
assert "index" in resource["description"].lower()
assert "json" in resource["format"].lower()
def test_hook(self):
testpackage = self.package
resource_dict = model_dictize.resource_dictize(self.resource,
{"model": model})
context = {
"model": model,
"session": model.Session,
"user": model.User.get("testsysadmin").name,
}
# no preview for type "plain text"
preview_url = self.preview_url
result = self.app.get(preview_url, status=409)
assert "No preview" in result.body, result.body
# no preview for type "ümlaut", should not fail
resource_dict["format"] = u"ümlaut"
l.action.update.resource_update(context, resource_dict)
result = self.app.get(preview_url, status=409)
assert "No preview" in result.body, result.body
resource_dict["format"] = "mock"
l.action.update.resource_update(context, resource_dict)
# there should be a preview for type "json"
preview_url = self.preview_url
result = self.app.get(preview_url, status=200)
assert "mock-preview" in result.body
assert "mock-preview.js" in result.body
assert self.plugin.calls["can_preview"] == 3, self.plugin.calls
assert (self.plugin.calls["setup_template_variables"] == 1
), self.plugin.calls
assert self.plugin.calls["preview_templates"] == 1, self.plugin.calls
# test whether the json preview is used
preview_url = h.url_for(
"resource.datapreview",
id=testpackage.id,
resource_id=testpackage.resources[1].id,
)
result = self.app.get(preview_url, status=200)
assert "mock-json-preview" in result.body
assert "mock-json-preview.js" in result.body
assert self.plugin.calls["can_preview"] == 4, self.plugin.calls
assert (self.plugin.calls["setup_template_variables"] == 1
), self.plugin.calls
assert self.plugin.calls["preview_templates"] == 1, self.plugin.calls
# def test_iframe_is_shown(self):
result = self.app.get(self.url)
assert 'data-module="data-viewer"' in result.body, result.body
assert "<iframe" in result.body, result.body
# def test_iframe_url_is_correct(self):
result = self.app.get(self.url)
assert str(self.preview_url) in result.body, (
self.preview_url,
result.body,
)
