def dictionary_update(self):
"""Update the dictionary for a given package."""
try:
if request.method == 'POST':
log.info("dictionary_update:POST:Content-Type:{0}".format(
request.content_type))
log.info("dictionary_update:request.body:{0}".format(
request.body))
self.update_data_dictionary(json.loads(request.body))
response.status_int = 200
response.headers['Content-Type'] = "application/json"
return json.dumps({"success": True})
else:
response.status_int = 501
response.headers['Content-Type'] = "application/json"
return json.dumps({
"success": False,
"error": {
"messsage": "Not Implemented"
}
})
except Exception as e:
response.status_int = 500
response.headers['Content-Type'] = "application/json"
log.error("dictionary_update:Exception: {0}".format(e.message))
return json.dumps({
"success": False,
"error": {
"messsage": "Exception"
}
})
def filtered_download(resource_view_id):
params = json.loads(request.form[u'params'])
resource_view = get_action(u'resource_view_show')(None, {
u'id': resource_view_id
})
search_text = text_type(params[u'search'][u'value'])
view_filters = resource_view.get(u'filters', {})
user_filters = text_type(params[u'filters'])
filters = merge_filters(view_filters, user_filters)
datastore_search = get_action(u'datastore_search')
unfiltered_response = datastore_search(
None, {
u"resource_id": resource_view[u'resource_id'],
u"limit": 0,
u"filters": view_filters,
})
cols = [f[u'id'] for f in unfiltered_response[u'fields']]
if u'show_fields' in resource_view:
cols = [c for c in cols if c in resource_view[u'show_fields']]
sort_list = []
for order in params[u'order']:
sort_by_num = int(order[u'column'])
sort_order = (u'desc' if order[u'dir'] == u'desc' else u'asc')
sort_list.append(cols[sort_by_num] + u' ' + sort_order)
cols = [c for (c, v) in zip(cols, params[u'visible']) if v]
colsearch_dict = {}
columns = params[u'columns']
for column in columns:
if column[u'search'][u'value']:
v = column[u'search'][u'value']
if v:
k = column[u'name']
# replace non-alphanumeric characters with FTS wildcard (_)
v = re.sub(r'[^0-9a-zA-Z\-]+', '_', v)
# append ':*' so we can do partial FTS searches
colsearch_dict[k] = v + u':*'
if colsearch_dict:
search_text = json.dumps(colsearch_dict)
else:
search_text = re.sub(r'[^0-9a-zA-Z\-]+', '_',
search_text) + u':*' if search_text else ''
return h.redirect_to(
h.url_for(u'datastore.dump', resource_id=resource_view[u'resource_id'])
+ u'?' + urlencode({
u'q': search_text,
u'plain': False,
u'language': u'simple',
u'sort': u','.join(sort_list),
u'filters': json.dumps(filters),
u'format': request.form[u'format'],
u'fields': u','.join(cols),
}))
def setup_template_variables(self, context, data_dict):
'''
Setup variables available to templates.
'''
# TODO: Apply variables to appropriate view
resource = data_dict['resource']
resource_view = data_dict['resource_view']
resource_view_id = resource_view.get('id', None)
# get the names of the fields on this resource in the datastore
fields = get_resource_datastore_fields(resource['id'])
# find all the views on this resource currently
views = toolkit.get_action('resource_view_list')(context, {
'id': resource['id']
})
# build a list of view options, adding a default view option of no view first
view_options = [{'text': toolkit._('(None)'), 'value': ''}]
# then loop through and add the other views
for view in views:
# but make sure we don't add this view to the list of options
if resource_view_id == view['id']:
continue
view_options.append({'text': view['title'], 'value': view['id']})
return {
'resource_json': json.dumps(resource),
'resource_view_json': json.dumps(resource_view),
'map_fields': [{
'text': field,
'value': field
} for field in fields],
'available_views': view_options,
'defaults': plugin_config,
'is_new': resource_view_id is None,
}
def test_02_own_activities_do_not_count_as_new(self):
'''Make a user do some activities and check that her own activities
don't increase her new activities count.'''
# The user has to view her dashboard activity stream first to mark any
# existing activities as read. For example when she follows a dataset
# below, past activities from the dataset (e.g. when someone created
# the dataset, etc.) will appear in her dashboard, and if she has never
# viewed her dashboard then those activities will be considered
# "unseen".
# We would have to do this if, when you follow something, you only get
# the activities from that object since you started following it, and
# not all its past activities as well.
self.dashboard_mark_activities_old(self.new_user)
# Create a new dataset.
params = json.dumps({
'name': 'my_new_package',
})
response = self.app.post(
'/api/action/package_create',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Follow a dataset.
params = json.dumps({'id': 'warandpeace'})
response = self.app.post(
'/api/action/follow_dataset',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Follow a user.
params = json.dumps({'id': 'annafan'})
response = self.app.post(
'/api/action/follow_user',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Follow a group.
params = json.dumps({'id': 'roger'})
response = self.app.post(
'/api/action/follow_group',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Update the dataset that we're following.
params = json.dumps({'name': 'warandpeace', 'notes': 'updated'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# User's own actions should not increase her activity count.
assert self.dashboard_new_activities_count(self.new_user) == 0
def top_tags(self):
response.content_type = 'application/json; charset=UTF-8'
stats = stats_lib.Stats()
top_tags = stats.top_tags()
data = {}
for tag, num_packages in top_tags:
data[tag.name] = str(num_packages)
data = {"top_tags" : json.dumps(data, ensure_ascii=False)}
return json.dumps(data)
def test_02_own_activities_do_not_count_as_new(self):
"""Make a user do some activities and check that her own activities
don't increase her new activities count."""
# The user has to view her dashboard activity stream first to mark any
# existing activities as read. For example when she follows a dataset
# below, past activities from the dataset (e.g. when someone created
# the dataset, etc.) will appear in her dashboard, and if she has never
# viewed her dashboard then those activities will be considered
# "unseen".
# We would have to do this if, when you follow something, you only get
# the activities from that object since you started following it, and
# not all its past activities as well.
self.dashboard_mark_activities_old(self.new_user)
# Create a new dataset.
params = json.dumps({"name": "my_new_package"})
response = self.app.post(
"/api/action/package_create", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Follow a dataset.
params = json.dumps({"id": "warandpeace"})
response = self.app.post(
"/api/action/follow_dataset", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Follow a user.
params = json.dumps({"id": "annafan"})
response = self.app.post(
"/api/action/follow_user", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Follow a group.
params = json.dumps({"id": "roger"})
response = self.app.post(
"/api/action/follow_group", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Update the dataset that we're following.
params = json.dumps({"name": "warandpeace", "notes": "updated"})
response = self.app.post(
"/api/action/package_update", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# User's own actions should not increase her activity count.
assert self.dashboard_new_activities_count(self.new_user) == 0
def setup_template_variables(self, context, data_dict):
# metadata = {'text_formats': self.text_formats,
# 'json_formats': self.json_formats,
# 'jsonp_formats': self.jsonp_formats,
# 'xml_formats': self.xml_formats}
url = proxy.get_proxified_resource_url(data_dict)
return {
# 'preview_metadata': json.dumps(metadata),
'resource_json': json.dumps(data_dict['resource']),
'resource_url': json.dumps(url)
}
def setup_template_variables(self, context, data_dict):
metadata = {'text_formats': self.text_formats,
'json_formats': self.json_formats,
'jsonp_formats': self.jsonp_formats,
'xml_formats': self.xml_formats}
url = proxy.get_proxified_resource_url(data_dict)
format_lower = data_dict['resource']['format'].lower()
if format_lower in self.jsonp_formats:
url = data_dict['resource']['url']
return {'preview_metadata': json.dumps(metadata),
'resource_json': json.dumps(data_dict['resource']),
'resource_url': json.dumps(url)}
def setup_template_variables(self, context, data_dict):
metadata = {'text_formats': self.text_formats,
'json_formats': self.json_formats,
'jsonp_formats': self.jsonp_formats,
'xml_formats': self.xml_formats}
url = proxy.get_proxified_resource_url(data_dict)
format_lower = data_dict['resource']['format'].lower()
if format_lower in self.jsonp_formats:
url = data_dict['resource']['url']
return {'preview_metadata': json.dumps(metadata),
'resource_json': json.dumps(data_dict['resource']),
'resource_url': json.dumps(url)}
def setup_template_variables(self, context, data_dict):
resource = data_dict['resource']
proxy_enabled = p.plugin_loaded('resource_proxy')
oauth2_enabled = p.plugin_loaded('oauth2')
same_domain = datapreview.on_same_domain(data_dict)
if 'oauth_req' not in resource:
oauth_req = 'false'
else:
oauth_req = resource['oauth_req']
url = resource['url']
if not check_query(resource):
details = "</br></br>This is not a ContextBroker query, please check <a href='https://forge.fiware.org/plugins/mediawiki/wiki/fiware/index.php/Publish/Subscribe_Broker_-_Orion_Context_Broker_-_User_and_Programmers_Guide'>Orion Context Broker documentation</a></br></br></br>"
f_details = "This is not a ContextBroker query, please check Orion Context Broker documentation."
h.flash_error(f_details, allow_html=False)
view_enable = [False, details]
elif not same_domain and not proxy_enabled:
details = "</br></br>Enable resource_proxy</br></br></br>"
f_details = "Enable resource_proxy."
h.flash_error(f_details, allow_html=False)
view_enable = [False, details]
url = ''
else:
if not same_domain:
url = self.get_proxified_ngsi_url(data_dict)
if oauth_req == 'true' and not p.toolkit.c.user:
details = "</br></br>In order to see this resource properly, you need to be logged in.</br></br></br>"
f_details = "In order to see this resource properly, you need to be logged in."
h.flash_error(f_details, allow_html=False)
view_enable = [False, details]
elif oauth_req == 'true' and not oauth2_enabled:
details = "</br></br>In order to see this resource properly, enable oauth2 extension</br></br></br>"
f_details = "In order to see this resource properly, enable oauth2 extension."
h.flash_error(f_details, allow_html=False)
view_enable = [False, details]
else:
data_dict['resource']['url'] = url
view_enable = [True, 'OK']
return {
'resource_json': json.dumps(data_dict['resource']),
'resource_url': json.dumps(url),
'view_enable': json.dumps(view_enable)
}
def _delete_vocabulary(self, vocab_id, user=None):
if user:
extra_environ = {'Authorization' : str(user.apikey)}
else:
extra_environ = None
params = {'id': vocab_id}
response = self._post('/api/action/vocabulary_delete', params=params,
extra_environ=extra_environ)
# Check the values of the response.
assert response['success'] == True
assert response['result'] is None
response['result']
# Get the list of vocabularies.
response = self._post('/api/action/vocabulary_list')
assert response['success'] == True
assert response['result']
# Check that the vocabulary we deleted is not in the list.
assert vocab_id not in [vocab['id'] for vocab in response['result']]
# Check that the deleted vocabulary can no longer be retrieved.
response = self.app.post('/api/action/vocabulary_show',
params=json.dumps(params),
extra_environ = {'Authorization':
str(self.sysadmin_user.apikey)},
status=404)
assert response.json['success'] == False
def build_style_cmd():
'''Generates a list of citation styles.
By default, the following styles will be shown first:
apa, modern-language-association, chicago-note-bibliography,
chicago-author-date, and ieee.
'''
major_styles = [
'apa',
'modern-language-association',
'chicago-note-bibliography',
'chicago-author-date',
'ieee',
'council-of-science-editors',
'american-medical-association',
'american-chemical-society',
'american-institute-of-physics',
'american-society-of-civil-engineers',
]
all_csl = os.listdir(csl_p)
major_csl = [s + '.csl' for s in major_styles if s + '.csl' in all_csl]
other_csl = [c for c in all_csl if c not in major_csl]
styles = build_styles(major_csl, 'major') + \
build_styles(other_csl, 'other')
styles_json = json.dumps(styles, separators=(',', ':'))
with open(os.path.join(p, 'csl_styles.json'), 'w') as f:
f.write(styles_json)
def call_action_api(app, action, apikey=None, status=200, **kwargs):
'''POST an HTTP request to the CKAN API and return the result.
Any additional keyword arguments that you pass to this function as **kwargs
are posted as params to the API.
Usage:
package_dict = post(app, 'package_create', apikey=apikey,
name='my_package')
assert package_dict['name'] == 'my_package'
num_followers = post(app, 'user_follower_count', id='annafan')
If you are expecting an error from the API and want to check the contents
of the error dict, you have to use the status param otherwise an exception
will be raised:
error_dict = post(app, 'group_activity_list', status=403,
id='invalid_id')
assert error_dict['message'] == 'Access Denied'
:param app: the test app to post to
:type app: paste.fixture.TestApp
:param action: the action to post to, e.g. 'package_create'
:type action: string
:param apikey: the API key to put in the Authorization header of the post
(optional, default: None)
:type apikey: string
:param status: the HTTP status code expected in the response from the CKAN
API, e.g. 403, if a different status code is received an exception will
be raised (optional, default: 200)
:type status: int
:param **kwargs: any other keyword arguments passed to this function will
be posted to the API as params
:raises paste.fixture.AppError: if the HTTP status code of the response
from the CKAN API is different from the status param passed to this
function
:returns: the 'result' or 'error' dictionary from the CKAN API response
:rtype: dictionary
'''
params = json.dumps(kwargs)
response = app.post('/api/action/{0}'.format(action), params=params,
extra_environ={'Authorization': str(apikey)}, status=status)
assert '/api/3/action/help_show?name={0}'.format(action) \
in response.json['help']
if status in (200,):
assert response.json['success'] is True
return response.json['result']
else:
assert response.json['success'] is False
return response.json['error']
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
same_domain = datapreview.on_same_domain(data_dict)
p.toolkit.c.gapi_key = h.config.get('ckanext.geoview.gapi.key')
if p.toolkit.check_ckan_version('2.3'):
proxy_url = proxy.get_proxified_resource_url(data_dict)
else:
proxy_url = data_dict['resource']['url']
if self.proxy_enabled and not same_domain:
proxy_url = proxy.get_proxified_resource_url(data_dict)
return {'proxy_service_url': json.dumps(get_proxified_service_url(data_dict)),
'proxy_url': json.dumps(proxy_url)}
def setup_template_variables(self, context, data_dict):
view_type = view_type = [('table', 'Table')]
widgets = get_widget(data_dict['resource_view'], view_type)
schema = datastore_fields_to_schema(data_dict['resource'])
filters = data_dict['resource_view'].get('filters', {})
data_dict['resource'].update({
'schema': {
'fields': schema
},
'title':
data_dict['resource']['name'],
'path':
data_dict['resource']['url'],
'api':
url_for('api.action',
ver=3,
logic_function='datastore_search',
resource_id=data_dict['resource']['id'],
filters=json.dumps(filters),
_external=True),
})
datapackage = {'resources': [data_dict['resource']]}
return {
'resource_view': data_dict['resource_view'],
'widgets': widgets,
'datapackage': datapackage
}
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
same_domain = on_same_domain(data_dict)
if not data_dict['resource'].get('format'):
data_dict['resource']['format'] = \
self._guess_format_from_extension(data_dict['resource']['url'])
if self.proxy_enabled and not same_domain:
proxy_url = proxy.get_proxified_resource_url(data_dict)
proxy_service_url = get_proxified_service_url(data_dict)
else:
proxy_url = data_dict['resource']['url']
proxy_service_url = data_dict['resource']['url']
gapi_key = config.get('ckanext.geoview.gapi_key')
if not p.toolkit.check_ckan_version(min_version='2.3'):
p.toolkit.c.resource['proxy_url'] = proxy_url
p.toolkit.c.resource['proxy_service_url'] = proxy_service_url
p.toolkit.c.resource['gapi_key'] = gapi_key
return {'resource_view_json': 'resource_view' in data_dict and json.dumps(data_dict['resource_view']),
'proxy_service_url': proxy_service_url,
'proxy_url': proxy_url,
'gapi_key': gapi_key,
'basemapsConfig' : self.basemapsConfig}
def setup_template_variables(self, context, data_dict):
"""Setup variables available to templates"""
#TODO: Apply variables to appropriate view.
datastore_fields = self._get_datastore_fields(data_dict['resource']['id'], context)
views = p.toolkit.get_action('resource_view_list')(context, {'id': data_dict['resource']['id']})
if 'id' in data_dict['resource_view']:
views = [v for v in views if v['id'] != data_dict['resource_view']['id']]
views = [{'text': _('(None)'), 'value': ''}] + [{'text': v['title'], 'value': v['id']} for v in views]
return {
'resource_json': json.dumps(data_dict['resource']),
'resource_view_json': json.dumps(data_dict['resource_view']),
'map_fields': [{'text': f, 'value': f} for f in datastore_fields],
'available_views': views,
'defaults': plugin_config,
'is_new': not('id' in data_dict['resource_view'])
}
def test_10_dashboard_activity_list_html_does_not_crash(self):
params = json.dumps({'name': 'irrelevant_dataset1'})
response = self.app.post('/api/action/package_create', params=params,
extra_environ={'Authorization': str(self.annafan['apikey'])})
assert response.json['success'] is True
params = json.dumps({'name': 'another_irrelevant_dataset'})
response = self.app.post('/api/action/package_create', params=params,
extra_environ={'Authorization': str(self.annafan['apikey'])})
assert response.json['success'] is True
res = self.app.get('/api/3/action/dashboard_activity_list_html',
extra_environ={'Authorization':
str(self.annafan['apikey'])})
assert res.json['success'] is True
def output_feed(self, results, feed_title, feed_description, feed_link,
feed_url, navigation_urls, feed_guid):
author_name = config.get('ckan.feeds.author_name', '').strip() or \
config.get('ckan.site_id', '').strip()
author_link = config.get('ckan.feeds.author_link', '').strip() or \
config.get('ckan.site_url', '').strip()
# TODO language
feed_class = None
for plugin in plugins.PluginImplementations(plugins.IFeed):
if hasattr(plugin, 'get_feed_class'):
feed_class = plugin.get_feed_class()
if not feed_class:
feed_class = _FixedAtom1Feed
feed = feed_class(
feed_title,
feed_link,
feed_description,
language=u'en',
author_name=author_name,
author_link=author_link,
feed_guid=feed_guid,
feed_url=feed_url,
previous_page=navigation_urls['previous'],
next_page=navigation_urls['next'],
first_page=navigation_urls['first'],
last_page=navigation_urls['last'],
)
for pkg in results:
additional_fields = {}
for plugin in plugins.PluginImplementations(plugins.IFeed):
if hasattr(plugin, 'get_item_additional_fields'):
additional_fields = plugin.get_item_additional_fields(pkg)
feed.add_item(
title=pkg.get('title', ''),
link=self.base_url +
h.url_for(controller='package', action='read', id=pkg['id']),
description=pkg.get('notes', ''),
updated=h.date_str_to_datetime(pkg.get('metadata_modified')),
published=h.date_str_to_datetime(pkg.get('metadata_created')),
unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']),
author_name=pkg.get('author', ''),
author_email=pkg.get('author_email', ''),
categories=[t['name'] for t in pkg.get('tags', [])],
enclosure=webhelpers.feedgenerator.Enclosure(
self.base_url + h.url_for(controller='api',
register='package',
action='show',
id=pkg['name'],
ver='2'),
unicode(len(json.dumps(pkg))), # TODO fix this
u'application/json'),
**additional_fields)
response.content_type = feed.mime_type
return feed.writeString('utf-8')
def test_04_activities_from_followed_datasets(self):
'''Activities from followed datasets should show in dashboard.'''
activities_before = self.dashboard_activity_list(self.new_user)
# Make someone else who new_user is not following update a dataset that
# new_user is following.
params = json.dumps({'name': 'warandpeace', 'notes': 'updated again'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.joeadmin['apikey'])})
assert response.json['success'] is True
# Check the new activity in new_user's dashboard.
activities = self.dashboard_activity_list(self.new_user)
new_activities = [
activity for activity in activities
if activity not in activities_before
]
assert len(new_activities) == 1
activity = new_activities[0]
assert activity['activity_type'] == 'changed package'
assert activity['user_id'] == self.joeadmin['id']
assert activity['data']['package']['name'] == 'warandpeace'
def setup_template_variables(self, context, data_dict):
view_type = view_type = [('simple', 'Chart')]
spec = {}
chart_type = data_dict['resource_view'].get('chart_type', False)
group = data_dict['resource_view'].get('group', False)
chart_series = data_dict['resource_view'].get('chart_series', False)
if chart_type:
spec.update({'type': chart_type})
if group:
spec.update({'group': group})
if chart_series:
spec.update({
'series':
chart_series
if isinstance(chart_series, list) else [chart_series]
})
widgets = get_widget(data_dict['resource_view'], view_type, spec)
filters = data_dict['resource_view'].get('filters', {})
limit = data_dict['resource_view'].get('limit', 100)
offset = data_dict['resource_view'].get('offset', 0)
self.datastore_schema = datastore_fields_to_schema(
data_dict['resource'])
data_dict['resource'].update({
'schema': {
'fields': self.datastore_schema
},
'title':
data_dict['resource']['name'],
'path':
data_dict['resource']['url'],
'api':
url_for('api.action',
ver=3,
logic_function='datastore_search',
resource_id=data_dict['resource']['id'],
filters=json.dumps(filters),
limit=limit,
offset=offset,
_external=True),
})
datapackage = {'resources': [data_dict['resource']]}
groups = valid_fields_as_options(self.datastore_schema)
chart_series = valid_fields_as_options(self.datastore_schema,
self.datastore_field_types)
return {
'resource_view': data_dict['resource_view'],
'widgets': widgets,
'datapackage': datapackage,
'chart_types': self.chart_types,
'chart_series': chart_series,
'groups': groups,
}
def show(self):
raw_data = self._get_data()
data = self._process(raw_data)
result = render('widget/3W/3W.html',
extra_vars={"data": json.dumps(data)})
return result
def duplicate_validator(key, data, errors, context):
if errors[key]:
return
value = json.loads(data[key])
unduplicated = list(set(value))
data[key] = json.dumps(unduplicated)
def getFields(self, context, data_dict):
url = self.getResourceURL(context, data_dict)
data = json.loads(urllib2.urlopen(url).read())
Dataframe = pd.read_json(json.dumps(data['result']['records']))
NumericColumns = (Dataframe.select_dtypes(
exclude=['object', 'datetime']).columns)
return NumericColumns
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
same_domain = on_same_domain(data_dict)
if not data_dict["resource"].get("format"):
data_dict["resource"][
"format"] = self._guess_format_from_extension(
data_dict["resource"]["url"])
if self.proxy_enabled and not same_domain:
proxy_url = proxy.get_proxified_resource_url(data_dict)
proxy_service_url = utils.get_proxified_service_url(data_dict)
else:
proxy_url = data_dict["resource"]["url"]
proxy_service_url = data_dict["resource"]["url"]
gapi_key = toolkit.config.get("ckanext.geoview.gapi_key")
return {
"resource_view_json":
"resource_view" in data_dict
and json.dumps(data_dict["resource_view"]),
"proxy_service_url":
proxy_service_url,
"proxy_url":
proxy_url,
"gapi_key":
gapi_key,
"basemapsConfig":
self.basemapsConfig,
}
def wikidata_keyword(key, data, errors, context):
"""
Accept wikidata keywords input in the following forms
and convert to a json list for storage:
1. a list of wikidata ids, eg.
["Q1", "Q2"]
2. a single comma separated string with wikidata ids, eg.
"Q1, Q2"
"""
if errors[key]:
return
value = data[key]
if value is not missing:
if isinstance(data[key], basestring):
value = [
element.strip() for element in value.split(',')
if element.strip()
]
if not isinstance(value, list):
errors[key].append(_('expecting list of strings'))
return
if not errors[key]:
data[key] = json.dumps(value)
return
def unicode_safe(value):
'''
Make sure value passed is treated as unicode, but don't raise
an error if it's not, just make a reasonable attempt to
convert other types passed.
This validator is a safer alternative to the old ckan idiom
of using the unicode() function as a validator. It tries
not to pollute values with Python repr garbage e.g. when passed
a list of strings (uses json format instead). It also
converts binary strings assuming either UTF-8 or CP1252
encodings (not ASCII, with occasional decoding errors)
'''
if isinstance(value, text_type):
return value
if hasattr(value, 'filename'):
# cgi.FieldStorage instance for uploaded files, show the name
value = value.filename
if value is missing or value is None:
return u''
if isinstance(value, bytes):
# bytes only arrive when core ckan or plugins call
# actions from Python code
try:
return value.decode(u'utf8')
except UnicodeDecodeError:
return value.decode(u'cp1252')
try:
return json.dumps(value, sort_keys=True, ensure_ascii=False)
except Exception:
# at this point we have given up. Just don't error out
try:
return text_type(value)
except Exception:
return u'\N{REPLACEMENT CHARACTER}'
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
same_domain = on_same_domain(data_dict)
if not data_dict['resource'].get('format'):
data_dict['resource']['format'] = \
self._guess_format_from_extension(data_dict['resource']['url'])
if self.proxy_enabled and not same_domain:
proxy_url = proxy.get_proxified_resource_url(data_dict)
proxy_service_url = get_proxified_service_url(data_dict)
else:
proxy_url = data_dict['resource']['url']
proxy_service_url = data_dict['resource']['url']
gapi_key = toolkit.config.get('ckanext.geoview.gapi_key')
if not toolkit.check_ckan_version(min_version='2.3'):
toolkit.c.resource['proxy_url'] = proxy_url
toolkit.c.resource['proxy_service_url'] = proxy_service_url
toolkit.c.resource['gapi_key'] = gapi_key
return {
'resource_view_json':
'resource_view' in data_dict
and json.dumps(data_dict['resource_view']),
'proxy_service_url':
proxy_service_url,
'proxy_url':
proxy_url,
'gapi_key':
gapi_key,
'basemapsConfig':
self.basemapsConfig
}
def call_action_api(app, action, apikey=None, status=200, **kwargs):
'''POST an HTTP request to the CKAN API and return the result.
Any additional keyword arguments that you pass to this function as **kwargs
are posted as params to the API.
Usage:
package_dict = post(app, 'package_create', apikey=apikey,
name='my_package')
assert package_dict['name'] == 'my_package'
num_followers = post(app, 'user_follower_count', id='annafan')
If you are expecting an error from the API and want to check the contents
of the error dict, you have to use the status param otherwise an exception
will be raised:
error_dict = post(app, 'group_activity_list', status=403,
id='invalid_id')
assert error_dict['message'] == 'Access Denied'
:param app: the test app to post to
:type app: paste.fixture.TestApp
:param action: the action to post to, e.g. 'package_create'
:type action: string
:param apikey: the API key to put in the Authorization header of the post
(optional, default: None)
:type apikey: string
:param status: the HTTP status code expected in the response from the CKAN
API, e.g. 403, if a different status code is received an exception will
be raised (optional, default: 200)
:type status: int
:param **kwargs: any other keyword arguments passed to this function will
be posted to the API as params
:raises paste.fixture.AppError: if the HTTP status code of the response
from the CKAN API is different from the status param passed to this
function
:returns: the 'result' or 'error' dictionary from the CKAN API response
:rtype: dictionary
'''
params = json.dumps(kwargs)
response = app.post('/api/action/{0}'.format(action), params=params,
extra_environ={'Authorization': str(apikey)}, status=status)
assert '/api/3/action/help_show?name={0}'.format(action) \
in response.json['help']
if status in (200,):
assert response.json['success'] is True
return response.json['result']
else:
assert response.json['success'] is False
return response.json['error']
def command(self):
if len(self.args) == 0:
self.parser.print_usage()
sys.exit(1)
cmd = self.args[0]
if cmd == 'build_styles':
major_styles = [
'apa',
'modern-language-association',
'chicago-note-bibliography',
'chicago-author-date',
'ieee',
'council-of-science-editors',
'american-medical-association',
'american-chemical-society',
'american-institute-of-physics',
'american-society-of-civil-engineers',
]
all_csl = os.listdir(self.csl_p)
major_csl = [
s + '.csl' for s in major_styles if s + '.csl' in all_csl
]
other_csl = [c for c in all_csl if c not in major_csl]
styles = self.build_styles(major_csl, 'major') + \
self.build_styles(other_csl, 'other')
styles_json = json.dumps(styles, separators=(',', ':'))
with open(os.path.join(self.p, 'csl_styles.json'), 'wb') as f:
f.write(styles_json)
def test_04_activities_from_datasets_of_followed_groups(self):
'''Activities from datasets of followed groups should show in the
dashboard.
'''
activities_before = self.dashboard_activity_list(self.new_user)
# Make someone that the user is not following update a dataset that the
# user is not following either, but that belongs to a group that the
# user is following.
params = json.dumps({'name': 'annakarenina', 'notes': 'updated'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.joeadmin['apikey'])})
assert response.json['success'] is True
# Check the new activity in new_user's dashboard.
activities = self.dashboard_activity_list(self.new_user)
new_activities = [
activity for activity in activities
if activity not in activities_before
]
assert len(new_activities) == 1
activity = new_activities[0]
assert activity['activity_type'] == 'changed package'
assert activity['user_id'] == self.joeadmin['id']
assert activity['data']['package']['name'] == 'annakarenina'
def unicode_safe(value):
'''
Make sure value passed is treated as unicode, but don't raise
an error if it's not, just make a reasonable attempt to
convert other types passed.
This validator is a safer alternative to the old ckan idiom
of using the unicode() function as a validator. It tries
not to pollute values with Python repr garbage e.g. when passed
a list of strings (uses json format instead). It also
converts binary strings assuming either UTF-8 or CP1252
encodings (not ASCII, with occasional decoding errors)
'''
if isinstance(value, text_type):
return value
if hasattr(value, 'filename'):
# cgi.FieldStorage instance for uploaded files, show the name
value = value.filename
if value is missing or value is None:
return u''
if isinstance(value, bytes):
# bytes only arrive when core ckan or plugins call
# actions from Python code
try:
return six.ensure_text(value)
except UnicodeDecodeError:
return value.decode(u'cp1252')
try:
return json.dumps(value, sort_keys=True, ensure_ascii=False)
except Exception:
# at this point we have given up. Just don't error out
try:
return text_type(value)
except Exception:
return u'\N{REPLACEMENT CHARACTER}'
def ajax(resource_view_id):
resource_view = get_action(u'resource_view_show'
)(None, {
u'id': resource_view_id
})
draw = int(request.form[u'draw'])
search_text = text_type(request.form[u'search[value]'])
offset = int(request.form[u'start'])
limit = int(request.form[u'length'])
view_filters = resource_view.get(u'filters', {})
user_filters = text_type(request.form[u'filters'])
filters = merge_filters(view_filters, user_filters)
datastore_search = get_action(u'datastore_search')
unfiltered_response = datastore_search(
None, {
u"resource_id": resource_view[u'resource_id'],
u"limit": 0,
u"filters": view_filters,
}
)
cols = [f[u'id'] for f in unfiltered_response[u'fields']]
if u'show_fields' in resource_view:
cols = [c for c in cols if c in resource_view[u'show_fields']]
sort_list = []
i = 0
while True:
if u'order[%d][column]' % i not in request.form:
break
sort_by_num = int(request.form[u'order[%d][column]' % i])
sort_order = (
u'desc' if request.form[u'order[%d][dir]' %
i] == u'desc' else u'asc'
)
sort_list.append(cols[sort_by_num] + u' ' + sort_order)
i += 1
response = datastore_search(
None, {
u"q": search_text,
u"resource_id": resource_view[u'resource_id'],
u"offset": offset,
u"limit": limit,
u"sort": u', '.join(sort_list),
u"filters": filters,
}
)
return json.dumps({
u'draw': draw,
u'iTotalRecords': unfiltered_response.get(u'total', 0),
u'iTotalDisplayRecords': response.get(u'total', 0),
u'aaData': [[text_type(row.get(colname, u''))
for colname in cols]
for row in response[u'records']],
})
def output_feed(self, results, feed_title, feed_description, feed_link,
feed_url, navigation_urls, feed_guid):
author_name = config.get('ckan.feeds.author_name', '').strip() or \
config.get('ckan.site_id', '').strip()
author_link = config.get('ckan.feeds.author_link', '').strip() or \
config.get('ckan.site_url', '').strip()
# TODO language
feed = _FixedAtom1Feed(
title=feed_title,
link=feed_link,
description=feed_description,
language=u'en',
author_name=author_name,
author_link=author_link,
feed_guid=feed_guid,
feed_url=feed_url,
previous_page=navigation_urls['previous'],
next_page=navigation_urls['next'],
first_page=navigation_urls['first'],
last_page=navigation_urls['last'],
)
for pkg in results:
updated_date = pkg.get('metadata_modified')
if updated_date:
updated_date = h.date_str_to_datetime(updated_date)
published_date = pkg.get('metadata_created')
if published_date:
published_date = h.date_str_to_datetime(published_date)
feed.add_item(
title=pkg.get('title', ''),
link=urlparse.urljoin(
self.base_url,
h.url_for(controller='package',
action='read',
id=pkg['id'])),
description=pkg.get('notes', ''),
updated=updated_date,
published=published_date,
unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']),
author_name=pkg.get('author', ''),
author_email=pkg.get('author_email', ''),
categories=[t['name'] for t in pkg.get('tags', [])],
enclosure=webhelpers.feedgenerator.Enclosure(
urlparse.urljoin(
self.base_url,
h.url_for(controller='api',
register='package',
action='show',
id=pkg['name'],
ver='2')),
unicode(len(json.dumps(pkg))), # TODO fix this
u'application/json'))
response.content_type = feed.mime_type
return feed.writeString('utf-8')
def composite_convert_to(key, data, errors, context):
unflat = unflatten(data)
for f in composite_convert_fields:
if f not in unflat:
continue
data[(f,)] = json.dumps(unflat[f], default=lambda x:None if x == missing else x)
convert_to_extras((f,), data, errors, context)
del data[(f,)]
def old_setup_template_variables(self, context, data_dict):
resource = data_dict["resource"]
package = tk.get_action("package_show")(data_dict={
"id": resource["package_id"]
})
pd_resource_id, dfe_resource_id = corresponding_resource_id(
resource["name"], package)
return {
'resource_json': json.dumps(data_dict['resource']),
'resource_view_json': json.dumps(data_dict['resource_view']),
'resource': resource,
'pd_resource_id': pd_resource_id,
'dfe_resource_id': dfe_resource_id,
'pd_params': json.dumps({}),
'dfe_params': json.dumps({}),
'dataset_id': package['id']
}
def _post(self, url, params=None, extra_environ=None):
if params is None:
params = {}
param_string = json.dumps(params)
response = self.app.post(url, params=param_string,
extra_environ=extra_environ)
assert not response.errors
return response.json
def append_time_period(key, data, errors, context):
if errors[key]:
return
value = data[key]
if data.get(('time_period',), ''):
out = json.loads(value)
out.append(data[('time_period',)])
data[key] = json.dumps(out)
def test_user_create_api_enabled_anon(self, app):
params = {
"name": "testinganewuseranon",
"email": "*****@*****.**",
"password": "******",
}
res = app.post("/api/3/action/user_create", json.dumps(params))
res_dict = res.json
assert res_dict["success"] is True
def setup_template_variables(self, context, data_dict):
metadata = {
"text_formats": self.text_formats,
"json_formats": self.json_formats,
"jsonp_formats": self.jsonp_formats,
"xml_formats": self.xml_formats,
}
url = proxy.get_proxified_resource_url(data_dict)
format_lower = data_dict["resource"]["format"].lower()
if format_lower in self.jsonp_formats:
url = data_dict["resource"]["url"]
return {
"preview_metadata": json.dumps(metadata),
"resource_json": json.dumps(data_dict["resource"]),
"resource_url": json.dumps(url),
}
def test_10_dashboard_activity_list_html_does_not_crash(self):
params = json.dumps({"name": "irrelevant_dataset1"})
response = self.app.post(
"/api/action/package_create", params=params, extra_environ={"Authorization": str(self.annafan["apikey"])}
)
assert response.json["success"] is True
params = json.dumps({"name": "another_irrelevant_dataset"})
response = self.app.post(
"/api/action/package_create", params=params, extra_environ={"Authorization": str(self.annafan["apikey"])}
)
assert response.json["success"] is True
res = self.app.get(
"/api/3/action/dashboard_activity_list_html", extra_environ={"Authorization": str(self.annafan["apikey"])}
)
assert res.json["success"] is True
def test_user_create_api_enabled_anon(self):
params = {
'name': 'testinganewuseranon',
'email': '*****@*****.**',
'password': '******',
}
res = self.app.post('/api/3/action/user_create', json.dumps(params))
res_dict = res.json
assert res_dict['success'] is True
def test_user_create_api_enabled_anon(self):
params = {
'name': 'testinganewuseranon',
'email': '*****@*****.**',
'password': '******',
}
res = self.app.post('/api/3/action/user_create', json.dumps(params))
res_dict = res.json
assert res_dict['success'] is True
def user_create(cls):
"""Create a new user."""
params = json.dumps({"name": "mr_new_user", "email": "*****@*****.**", "password": "******"})
response = cls.app.post(
"/api/action/user_create", params=params, extra_environ={"Authorization": str(cls.testsysadmin["apikey"])}
)
assert response.json["success"] is True
new_user = response.json["result"]
return new_user
def test_vocabulary_delete_not_logged_in(self):
'''Test that users who are not logged in cannot delete vocabularies.'''
params = {'id': self.genre_vocab['id']}
param_string = json.dumps(params)
response = self.app.post('/api/action/vocabulary_delete',
params=param_string,
status=403)
assert response.json['success'] == False
assert response.json['error']['message'] == 'Access denied'
def _addExtras(self, package_dict, extras):
extras_as_dict = []
for key, value in extras.iteritems():
if isinstance(value, (list, dict)):
extras_as_dict.append({'key': key, 'value': json.dumps(value)})
else:
extras_as_dict.append({'key': key, 'value': value})
package_dict['extras'] = extras_as_dict
def to_jsonp(data):
content_type = 'application/json;charset=utf-8'
result = json.dumps(data, sort_keys=True)
if 'callback' in request.params:
response.headers['Content-Type'] = content_type
cbname = request.params['callback']
result = '%s(%s);' % (cbname, result)
else:
response.headers['Content-Type'] = content_type
return result
def test_user_create_api_disabled(self):
params = {
'name': 'testinganewuser',
'email': '*****@*****.**',
'password': '******',
}
res = self.app.post('/api/3/action/user_create', json.dumps(params),
expect_errors=True)
res_dict = res.json
assert res_dict['success'] is False
def to_jsonp(data):
content_type = 'application/json;charset=utf-8'
result = json.dumps(data, sort_keys=True)
if 'callback' in request.params:
response.headers['Content-Type'] = content_type
cbname = request.params['callback']
result = '%s(%s);' % (cbname, result)
else:
response.headers['Content-Type'] = content_type
return result
def test_vocabulary_update_none_tags(self):
'''Test updating vocabularies with None for 'tags'.
'''
params = {'id': self.genre_vocab['id'], 'tags': None}
response = self.app.post('/api/action/vocabulary_update',
params=json.dumps(params),
extra_environ = {'Authorization':
str(self.sysadmin_user.apikey)},
status=400)
assert "Integrity Error" in response.body, response.body
def test_vocabulary_delete_not_authorized(self):
'''Test that users who are not authorized cannot delete vocabs.'''
params = {'id': self.genre_vocab['id']}
param_string = json.dumps(params)
response = self.app.post('/api/action/vocabulary_delete',
params=param_string,
extra_environ = {'Authorization':
str(self.normal_user.apikey)},
status=403)
assert response.json['success'] == False
assert response.json['error']['message'] == 'Access denied'
def test_add_tag_not_logged_in(self):
tag_dict = {
'name': 'noise',
'vocabulary_id': self.genre_vocab['id']
}
tag_string = json.dumps(tag_dict)
response = self.app.post('/api/action/tag_create',
params=tag_string,
status=403)
assert response.json['success'] == False
assert response.json['error']['message'] == 'Access denied'
def test_vocabulary_update_no_id(self):
params = {'name': 'bagel radio'}
param_string = json.dumps(params)
response = self.app.post('/api/action/vocabulary_update',
params=param_string,
extra_environ = {'Authorization':
str(self.sysadmin_user.apikey)},
status=409)
assert response.json['success'] == False
assert response.json['error'].has_key('id')
assert response.json['error']['id'] == 'id not in data'
def test_vocabulary_create_not_logged_in(self):
'''Test that users who are not logged in cannot create vocabularies.'''
params = {'name':
"Spam Vocabulary: SpamCo Duck Rental: Rent Your Ducks From Us!"}
param_string = json.dumps(params)
response = self.app.post('/api/action/vocabulary_create',
params=param_string,
status=403)
assert response.json['success'] == False
assert response.json['error']['message'] == 'Access denied'
def test_08_maximum_number_of_new_activities(self):
'''Test that the new activities count does not go higher than 15, even
if there are more than 15 new activities from the user's followers.'''
for n in range(0, 20):
notes = "Updated {n} times".format(n=n)
params = json.dumps({'name': 'warandpeace', 'notes': notes})
response = self.app.post('/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.joeadmin['apikey'])})
assert response.json['success'] is True
assert self.dashboard_new_activities_count(self.new_user) == 15
def test_vocabulary_create_none_tags(self):
'''Test creating new vocabularies with None for 'tags'.
'''
params = {'name': 'foobar', 'tags': None}
response = self.app.post('/api/action/vocabulary_create',
params=json.dumps(params),
extra_environ = {'Authorization':
str(self.sysadmin_user.apikey)},
status=400)
assert "Integrity Error" in response.body
def test_user_create_api_enabled_sysadmin(self):
params = {
'name': 'testinganewusersysadmin',
'email': '*****@*****.**',
'password': '******',
}
res = self.app.post(
'/api/3/action/user_create',
json.dumps(params),
extra_environ={'Authorization': str(self.sysadmin_user.apikey)})
res_dict = res.json
assert res_dict['success'] is True
def user_create(cls):
'''Create a new user.'''
params = json.dumps({
'name': 'mr_new_user',
'email': '*****@*****.**',
'password': '******',
})
response = cls.app.post('/api/action/user_create', params=params,
extra_environ={'Authorization': str(cls.testsysadmin['apikey'])})
assert response.json['success'] is True
new_user = response.json['result']
return new_user
