def new(self, data=None, errors=None, error_summary=None):
package_type = self._guess_package_type(True)
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params,}
# Package needs to have a organization group in the call to check_access
# and also to save it
try:
check_access('package_create',context)
except NotAuthorized:
abort(401, _('Unauthorized to create a package'))
if context['save'] and not data:
return self._save_new(context)
data = data or clean_dict(unflatten(tuplize_dict(parse_params(
request.params, ignore_keys=[CACHE_PARAMETER]))))
c.resources_json = json.dumps(data.get('resources',[]))
errors = errors or {}
error_summary = error_summary or {}
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.errors_json = json.dumps(errors)
self._setup_template_variables(context, {'id': id})
# TODO: This check is to maintain backwards compatibility with the old way of creating
# custom forms. This behaviour is now deprecated.
if hasattr(self, 'package_form'):
c.form = render(self.package_form, extra_vars=vars)
else:
c.form = render(self._package_form(package_type=package_type), extra_vars=vars)
return render( self._new_template(package_type))
def test_04_new_activities_count(self):
"""Test that new activities from objects that a user follows increase
her new activities count."""
# Make someone else who new_user is not following update a dataset that
# new_user is following.
params = json.dumps({"name": "warandpeace", "notes": "updated again"})
response = self.app.post(
"/api/action/package_update", params=params, extra_environ={"Authorization": str(self.joeadmin["apikey"])}
)
assert response.json["success"] is True
# Make someone that the user is following create a new dataset.
params = json.dumps({"name": "annas_new_dataset"})
response = self.app.post(
"/api/action/package_create", params=params, extra_environ={"Authorization": str(self.annafan["apikey"])}
)
assert response.json["success"] is True
# Make someone that the user is not following update a dataset that
# the user is not following, but that belongs to a group that the user
# is following.
params = json.dumps({"name": "annakarenina", "notes": "updated"})
response = self.app.post(
"/api/action/package_update",
params=params,
extra_environ={"Authorization": str(self.testsysadmin["apikey"])},
)
assert response.json["success"] is True
# FIXME: The number here should be 3 but activities from followed
# groups are not appearing in dashboard. When that is fixed, fix this
# number.
assert self.dashboard_new_activities_count(self.new_user) == 2
def resource_embedded_dataviewer(self, id, resource_id, width=500, height=500):
"""
Embeded page for a read-only resource dataview. Allows
for width and height to be specified as part of the
querystring (as well as accepting them via routes).
"""
context = {"model": model, "session": model.Session, "user": c.user or c.author}
try:
c.resource = get_action("resource_show")(context, {"id": resource_id})
c.package = get_action("package_show")(context, {"id": id})
c.resource_json = json.dumps(c.resource)
# double check that the resource belongs to the specified package
if not c.resource["id"] in [r["id"] for r in c.package["resources"]]:
raise NotFound
except NotFound:
abort(404, _("Resource not found"))
except NotAuthorized:
abort(401, _("Unauthorized to read resource %s") % id)
# Construct the recline state
state_version = int(request.params.get("state_version", "1"))
recline_state = self._parse_recline_state(request.params)
if recline_state is None:
abort(400, ('"state" parameter must be a valid recline ' "state (version %d)" % state_version))
c.recline_state = json.dumps(recline_state)
c.width = max(int(request.params.get("width", width)), 100)
c.height = max(int(request.params.get("height", height)), 100)
c.embedded = True
return render("package/resource_embedded_dataviewer.html")
def test_04_new_activities_count(self):
'''Test that new activities from objects that a user follows increase
her new activities count.'''
# Make someone else who new_user is not following update a dataset that
# new_user is following.
params = json.dumps({'name': 'warandpeace', 'notes': 'updated again'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.joeadmin['apikey'])})
assert response.json['success'] is True
# Make someone that the user is following create a new dataset.
params = json.dumps({'name': 'annas_new_dataset'})
response = self.app.post(
'/api/action/package_create',
params=params,
extra_environ={'Authorization': str(self.annafan['apikey'])})
assert response.json['success'] is True
# Make someone that the user is not following update a dataset that
# the user is not following, but that belongs to a group that the user
# is following.
params = json.dumps({'name': 'annakarenina', 'notes': 'updated'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.testsysadmin['apikey'])})
assert response.json['success'] is True
# FIXME: The number here should be 3 but activities from followed
# groups are not appearing in dashboard. When that is fixed, fix this
# number.
assert self.dashboard_new_activities_count(self.new_user) == 2
def _create_task(self, dataset):
import ckan.lib.celery_app as celery_app
send_task = celery_app.celery.send_task
site_user = t.get_action('get_site_user')({
'model': model,
'ignore_auth': True,
'defer_commit': True
}, {})
context = json.dumps({
'site_url':
self.site_url,
'site_user_apikey':
site_user['apikey'],
'spatial_datastore_jdbc_url':
self.spatial_datastore_jdbc_url,
'spatial_ingester_filepath':
self.spatial_ingester_filepath,
})
dataset_dict = package_dictize(dataset, {'model': model})
data = json.dumps(dataset_dict)
task_id = make_uuid()
queue = 'priority'
send_task("os.spatial_ingest",
args=[context, data],
task_id=task_id,
queue=queue)
log.debug(
'Spatial Ingest put into celery queue %s: %s site_user=%s site_url=%s',
queue, dataset.name, site_user['name'], self.site_url)
def resource_embedded_dataviewer(self, id, resource_id):
"""
Embeded page for a read-only resource dataview.
"""
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
c.resource = get_action('resource_show')(context, {'id': resource_id})
c.package = get_action('package_show')(context, {'id': id})
c.resource_json = json.dumps(c.resource)
# double check that the resource belongs to the specified package
if not c.resource['id'] in [ r['id'] for r in c.package['resources'] ]:
raise NotFound
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
# Construct the recline state
state_version = int(request.params.get('state_version', '1'))
recline_state = self._parse_recline_state(request.params)
if recline_state is None:
abort(400, ('"state" parameter must be a valid recline state (version %d)' % state_version))
c.recline_state = json.dumps(recline_state)
c.width = max(int(request.params.get('width', 500)), 100)
c.height = max(int(request.params.get('height', 500)), 100)
c.embedded = True
return render('package/resource_embedded_dataviewer.html')
def test_02_own_activities_do_not_count_as_new(self):
'''Make a user do some activities and check that her own activities
don't increase her new activities count.'''
# The user has to view her dashboard activity stream first to mark any
# existing activities as read. For example when she follows a dataset
# below, past activities from the dataset (e.g. when someone created
# the dataset, etc.) will appear in her dashboard, and if she has never
# viewed her dashboard then those activities will be considered
# "unseen".
# We would have to do this if, when you follow something, you only get
# the activities from that object since you started following it, and
# not all its past activities as well.
self.dashboard_mark_all_new_activities_as_old(self.new_user)
# Create a new dataset.
params = json.dumps({
'name': 'my_new_package',
})
response = self.app.post(
'/api/action/package_create',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Follow a dataset.
params = json.dumps({'id': 'warandpeace'})
response = self.app.post(
'/api/action/follow_dataset',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Follow a user.
params = json.dumps({'id': 'annafan'})
response = self.app.post(
'/api/action/follow_user',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Follow a group.
params = json.dumps({'id': 'roger'})
response = self.app.post(
'/api/action/follow_group',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# Update the dataset that we're following.
params = json.dumps({'name': 'warandpeace', 'notes': 'updated'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.new_user['apikey'])})
assert response.json['success'] is True
# User's own actions should not increase her activity count.
assert self.dashboard_new_activities_count(self.new_user) == 0
def test_01_unfollow_user_missing_object_id(self):
for id in ('missing', None, ''):
if id == 'missing':
params = json.dumps({})
else:
params = json.dumps({'id':id})
extra_environ = {'Authorization': str(self.annafan.apikey),}
response = self.app.post('/api/action/unfollow_user',
params=params, extra_environ=extra_environ, status=409).json
assert response['success'] == False
assert response['error']['id'] == [u'Missing value']
def test_02_own_activities_do_not_count_as_new(self):
"""Make a user do some activities and check that her own activities
don't increase her new activities count."""
# The user has to view her dashboard activity stream first to mark any
# existing activities as read. For example when she follows a dataset
# below, past activities from the dataset (e.g. when someone created
# the dataset, etc.) will appear in her dashboard, and if she has never
# viewed her dashboard then those activities will be considered
# "unseen".
# We would have to do this if, when you follow something, you only get
# the activities from that object since you started following it, and
# not all its past activities as well.
self.dashboard_mark_activities_old(self.new_user)
# Create a new dataset.
params = json.dumps({"name": "my_new_package"})
response = self.app.post(
"/api/action/package_create", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Follow a dataset.
params = json.dumps({"id": "warandpeace"})
response = self.app.post(
"/api/action/follow_dataset", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Follow a user.
params = json.dumps({"id": "annafan"})
response = self.app.post(
"/api/action/follow_user", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Follow a group.
params = json.dumps({"id": "roger"})
response = self.app.post(
"/api/action/follow_group", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# Update the dataset that we're following.
params = json.dumps({"name": "warandpeace", "notes": "updated"})
response = self.app.post(
"/api/action/package_update", params=params, extra_environ={"Authorization": str(self.new_user["apikey"])}
)
assert response.json["success"] is True
# User's own actions should not increase her activity count.
assert self.dashboard_new_activities_count(self.new_user) == 0
def test_04_am_following_user_missing_id(self):
for id in ('missing', None, ''):
if id == 'missing':
params = json.dumps({})
else:
params = json.dumps({'id': id})
extra_environ = {'Authorization': str(self.annafan.apikey)}
response = self.app.post('/api/action/am_following_user',
params=params,
extra_environ=extra_environ,
status=409).json
assert response['success'] is False
assert response['error']['id'] == [u'Missing value']
def resource_embedded_dataviewer(self,
id,
resource_id,
width=500,
height=500):
"""
Embeded page for a read-only resource dataview. Allows
for width and height to be specified as part of the
querystring (as well as accepting them via routes).
"""
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
c.resource = get_action('resource_show')(context, {
'id': resource_id
})
c.package = get_action('package_show')(context, {'id': id})
c.resource_json = json.dumps(c.resource)
# double check that the resource belongs to the specified package
if not c.resource['id'] in [
r['id'] for r in c.package['resources']
]:
raise NotFound
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
# Construct the recline state
state_version = int(request.params.get('state_version', '1'))
recline_state = self._parse_recline_state(request.params)
if recline_state is None:
abort(
400,
('"state" parameter must be a valid recline state (version %d)'
% state_version))
c.recline_state = json.dumps(recline_state)
c.width = max(int(request.params.get('width', width)), 100)
c.height = max(int(request.params.get('height', height)), 100)
c.embedded = True
return render('package/resource_embedded_dataviewer.html')
def gather_stage(self, harvest_job):
log.debug('In SNLHarvester gather_stage')
metadata_path = self._fetch_metadata_file()
ids = []
try:
parser = MetaDataParser(metadata_path)
for dataset in parser.list_datasets():
metadata = parser.parse_set(dataset)
metadata['translations'].extend(
self._metadata_term_translations()
)
log.debug(metadata)
obj = HarvestObject(
guid=metadata['id'],
job=harvest_job,
content=json.dumps(metadata)
)
obj.save()
log.debug('adding ' + metadata['id'] + ' to the queue')
ids.append(obj.id)
finally:
temp_dir = os.path.dirname(metadata_path)
log.debug('Deleting directory ' + temp_dir)
shutil.rmtree(temp_dir)
return ids
def resource_read(self, id, resource_id):
context = {"model": model, "session": model.Session, "user": c.user or c.author}
try:
c.resource = get_action("resource_show")(context, {"id": resource_id})
c.package = get_action("package_show")(context, {"id": id})
# required for nav menu
c.pkg = context["package"]
c.resource_json = json.dumps(c.resource)
c.pkg_dict = c.package
except NotFound:
abort(404, _("Resource not found"))
except NotAuthorized:
abort(401, _("Unauthorized to read resource %s") % id)
# get package license info
license_id = c.package.get("license_id")
try:
c.package["isopen"] = model.Package.get_license_register()[license_id].isopen()
except KeyError:
c.package["isopen"] = False
# TODO: find a nicer way of doing this
c.datastore_api = "%s/api/action" % config.get("ckan.site_url", "").rstrip("/")
c.related_count = c.pkg.related_count
return render("package/resource_read.html")
def test_01_user_follow_dataset_missing_apikey(self):
params = json.dumps({'id': self.warandpeace.id})
response = self.app.post('/api/action/follow_dataset',
params=params,
status=403).json
assert response['success'] == False
assert response['error']['message'] == 'Access denied'
def read_ajax(self, id, revision=None):
package_type = self._get_package_type(id)
context = {
"model": model,
"session": model.Session,
"user": c.user or c.author,
"extras_as_string": True,
"schema": self._form_to_db_schema(package_type=package_type),
"revision_id": revision,
}
try:
data = get_action("package_show")(context, {"id": id})
schema = self._db_to_form_schema(package_type=package_type)
if schema:
data, errors = validate(data, schema)
except NotAuthorized:
abort(401, _("Unauthorized to read package %s") % "")
except NotFound:
abort(404, _("Dataset not found"))
## hack as db_to_form schema should have this
data["tag_string"] = ", ".join([tag["name"] for tag in data.get("tags", [])])
data.pop("tags")
data = flatten_to_string_key(data)
response.headers["Content-Type"] = "application/json;charset=utf-8"
return json.dumps(data)
def history_ajax(self, id):
context = {"model": model, "session": model.Session, "user": c.user or c.author, "extras_as_string": True}
data_dict = {"id": id}
try:
pkg_revisions = get_action("package_revision_list")(context, data_dict)
except NotAuthorized:
abort(401, _("Unauthorized to read package %s") % "")
except NotFound:
abort(404, _("Dataset not found"))
data = []
approved = False
for num, revision in enumerate(pkg_revisions):
if not approved and revision["approved_timestamp"]:
current_approved, approved = True, True
else:
current_approved = False
data.append(
{
"revision_id": revision["id"],
"message": revision["message"],
"timestamp": revision["timestamp"],
"author": revision["author"],
"approved": bool(revision["approved_timestamp"]),
"current_approved": current_approved,
}
)
response.headers["Content-Type"] = "application/json;charset=utf-8"
return json.dumps(data)
def _json_encode_attributes(self, properties):
attributes = []
for key, value in properties:
if value:
attributes.append((key, value))
return json.dumps(attributes)
def test_04_activities_from_followed_datasets(self):
'''Activities from followed datasets should show in dashboard.'''
activities_before = self.dashboard_activity_list(self.new_user)
# Make someone else who new_user is not following update a dataset that
# new_user is following.
params = json.dumps({'name': 'warandpeace', 'notes': 'updated again'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.joeadmin['apikey'])})
assert response.json['success'] is True
# Check the new activity in new_user's dashboard.
activities = self.dashboard_activity_list(self.new_user)
new_activities = [
activity for activity in activities
if activity not in activities_before
]
assert len(new_activities) == 1
activity = new_activities[0]
assert activity['activity_type'] == 'changed package'
assert activity['user_id'] == self.joeadmin['id']
assert activity['data']['package']['name'] == 'warandpeace'
def fetch_stage(self, harvest_object):
log.debug('In fetch_stage ' + self.info()['title'])
d = json.loads(harvest_object.content)
d = self.scraper.fetch(d)
harvest_object.content = json.dumps(d)
harvest_object.save()
return True
def import_stage(self,harvest_object):
log.debug('In SRDAHarvester import_stage')
if not harvest_object:
log.error('No harvest object received')
return False
if harvest_object.content is None:
self._save_object_error('Empty content for object %s' % harvest_object.id,
harvest_object, 'Import')
return False
#self._set_config(harvest_object.job.source.config)
try:
package_dict = json.loads(harvest_object.content)
package_dict["id"] = harvest_object.guid
package_dict["extras"][u"資料庫名稱"] = u'SRDA'
package_dict["extras"][u"資料庫網址"] = u'http://srda.sinica.edu.tw/'
#print package_dict
for key in package_dict['extras'].keys():
if not isinstance(package_dict['extras'][key], basestring):
try:
package_dict['extras'][key] = json.dumps(package_dict['extras'][key])
except TypeError:
# If converting to a string fails, just delete it.
del package_dict['extras'][key]
result = self._create_or_update_package(package_dict,harvest_object)
return True
except ValidationError,e:
self._save_object_error('Invalid package with GUID %s: %r' % (harvest_object.guid, e.error_dict),
harvest_object, 'Import')
def test_edit_package(self):
# create the package to be edited
pkg_name = 'test4'
test_pkg = self.get_package_fixture(pkg_name)
pkg = CreateTestData.create_arbitrary(test_pkg)
# edit it
offset = '/api/rest/package/%s' % pkg_name
edited_pkg = copy.deepcopy(test_pkg)
edited_pkg['title'] = 'Edited title'
postparams = '%s=1' % json.dumps(edited_pkg)
result = self.app.put(offset, postparams, status=[200], extra_environ=self.extra_environ_sysadmin)
# check returned dict is correct
res = json.loads(result.body)
assert_equal(res['name'], test_pkg['name'])
assert res['id']
assert_equal(res['title'], 'Edited title')
assert_equal(res['license_id'], test_pkg['license_id'])
assert res['organization']['name'] == test_pkg['groups'][0]
assert_equal(res['extras'].get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(res['resources'][0].get('description'), test_pkg['resources'][0]['description'])
assert_equal(set(res['tags']), set(test_pkg['tags']))
# check package was edited ok
pkg = model.Package.by_name(test_pkg['name'])
pkg_dict = get_action('package_show')(self.context, {'id': test_pkg['name']})
assert_equal(pkg.name, test_pkg['name'])
assert_equal(pkg.title, 'Edited title')
assert pkg.get_organization().name == test_pkg['groups'][0]
assert_equal(pkg.extras.get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(pkg.resources[0].description, test_pkg['resources'][0]['description'])
assert_equal(set([tag['name'] for tag in pkg_dict['tags']]), set(test_pkg['tags']))
def test_edit_package(self):
# create the package to be edited
pkg_name = 'test4'
test_pkg = self.get_package_fixture(pkg_name)
pkg = CreateTestData.create_arbitrary(test_pkg)
# edit it
offset = '/api/rest/package/%s' % pkg_name
edited_pkg = copy.deepcopy(test_pkg)
edited_pkg['title'] = 'Edited title'
postparams = '%s=1' % json.dumps(edited_pkg)
result = self.app.put(offset, postparams, status=[200], extra_environ=self.extra_environ_sysadmin)
# check returned dict is correct
res = json.loads(result.body)
assert_equal(res['name'], test_pkg['name'])
assert res['id']
assert_equal(res['title'], 'Edited title')
assert_equal(res['license_id'], test_pkg['license_id'])
assert res['organization']['name'] == test_pkg['groups'][0]
assert_equal(res['extras'].get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(res['resources'][0].get('description'), test_pkg['resources'][0]['description'])
assert_equal(set(res['tags']), set(test_pkg['tags']))
# check package was edited ok
pkg = model.Package.by_name(test_pkg['name'])
pkg_dict = get_action('package_show')(self.context, {'id': test_pkg['name']})
assert_equal(pkg.name, test_pkg['name'])
assert_equal(pkg.title, 'Edited title')
assert pkg.get_organization().name == test_pkg['groups'][0]
assert_equal(pkg.extras.get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(pkg.resources[0].description, test_pkg['resources'][0]['description'])
assert_equal(set([tag['name'] for tag in pkg_dict['tags']]), set(test_pkg['tags']))
def data(self):
# Get the Europe dataset
rootdir = get_root_dir()
data_file = os.path.join(rootdir, 'ckanext', 'pdeu', 'data', 'eu.json')
f = open(data_file, 'r')
o = json.load(f)
# Get the package count by country
q = Session.query(
distinct(PackageExtra.value),
func.count(PackageExtra.value)
).\
filter(PackageExtra.key == u'eu_country').\
group_by(PackageExtra.value)
values = dict(q.all())
# Set the package count for each country
for ft in o['features']:
code = ft['properties']['NUTS']
ft['properties']['packages'] = (values.get(code, 0))
response.content_type = 'application/json'
response.pragma = None
response.cache_control = 'public; max-age: 3600'
response.cache_expires(seconds=3600)
return json.dumps(o)
def test_04_activities_from_datasets_of_followed_groups(self):
'''Activities from datasets of followed groups should show in the
dashboard.
'''
activities_before = self.dashboard_activity_list(self.new_user)
# Make someone that the user is not following update a dataset that the
# user is not following either, but that belongs to a group that the
# user is following.
params = json.dumps({'name': 'annakarenina', 'notes': 'updated'})
response = self.app.post(
'/api/action/package_update',
params=params,
extra_environ={'Authorization': str(self.testsysadmin['apikey'])})
assert response.json['success'] is True
# Check the new activity in new_user's dashboard.
activities = self.dashboard_activity_list(self.new_user)
new_activities = [
activity for activity in activities
if activity not in activities_before
]
assert len(new_activities) == 1
activity = new_activities[0]
assert activity['activity_type'] == 'changed package'
assert activity['user_id'] == self.testsysadmin['id']
assert activity['data']['package']['name'] == 'annakarenina'
def test_04_activities_from_followed_groups(self):
'''Activities from followed groups should show in the dashboard.'''
activities_before = self.dashboard_activity_list(self.new_user)
# Make someone that the user is not following update a group that the
# user is following.
params = json.dumps({'id': 'roger', 'description': 'updated'})
response = self.app.post(
'/api/action/group_update',
params=params,
extra_environ={'Authorization': str(self.testsysadmin['apikey'])})
assert response.json['success'] is True
# Check the new activity in new_user's dashboard.
activities = self.dashboard_activity_list(self.new_user)
new_activities = [
activity for activity in activities
if activity not in activities_before
]
assert len(new_activities) == 1
activity = new_activities[0]
assert activity['activity_type'] == 'changed group'
assert activity['user_id'] == self.testsysadmin['id']
assert activity['data']['group']['name'] == 'roger'
def get_metadata_provenance(self,
harvest_object,
harvested_provenance=None):
'''Returns the metadata_provenance for a dataset, which is the details
of this harvest added onto any existing metadata_provenance value in
the dataset. This should be stored in the metadata_provenance extra
when harvesting.
Provenance is a record of harvests, imports and perhaps other
activities of production too, as suggested by W3C PROV.
This helps keep track when a dataset is created in site A, imported
into site B, harvested into site C and from there is harvested into
site D. The metadata_provence will be a list of four dicts with the
details: [A, B, C, D].
'''
reharvest = True
if isinstance(harvested_provenance, basestring):
harvested_provenance = json.loads(harvested_provenance)
elif harvested_provenance is None:
harvested_provenance = []
reharvest = False
metadata_provenance = harvested_provenance + \
[self.get_metadata_provenance_for_just_this_harvest(
harvest_object,
reharvest
)]
return json.dumps(metadata_provenance)
def read_ajax(self, id, revision=None):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,
'schema': self._form_to_db_schema(),
'revision_id': revision
}
try:
data = get_action('package_show')(context, {'id': id})
schema = self._db_to_form_schema()
if schema:
data, errors = validate(data, schema)
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Package not found'))
## hack as db_to_form schema should have this
data['tag_string'] = ', '.join(
[tag['name'] for tag in data.get('tags', [])])
data.pop('tags')
data = flatten_to_string_key(data)
response.headers['Content-Type'] = 'application/json;charset=utf-8'
return json.dumps(data)
def edit(self, id, data=None, errors=None, error_summary=None):
package_type = self._get_package_type(id)
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params,
'moderated': config.get('moderated'),
'pending': True,}
if context['save'] and not data:
return self._save_edit(id, context)
try:
c.pkg_dict = get_action('package_show')(context, {'id':id})
context['for_edit'] = True
old_data = get_action('package_show')(context, {'id':id})
# old data is from the database and data is passed from the
# user if there is a validation error. Use users data if there.
data = data or old_data
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Dataset not found'))
c.pkg = context.get("package")
c.resources_json = json.dumps(data.get('resources',[]))
try:
check_access('package_update',context)
except NotAuthorized, e:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
def test_04_dataset_follower_list_missing_id(self):
params = json.dumps({})
response = self.app.post('/api/action/dataset_follower_list',
params=params,
status=409).json
assert response['success'] is False
assert response['error']['id'] == ['Missing value']
def generate(cls, xmlrpc_settings):
drupal = DrupalClient(xmlrpc_settings)
orgs = {}
has_errors = False
orgs_to_lookup = set()
orgs_to_lookup.add('Northern Ireland Executive')
for org_name in orgs_to_lookup:
org_name = canonise_organisation_name(org_name)
org_id = drupal.match_organisation(org_name)
if org_id == False:
log.error('Could not find organisation %r', org_name)
has_errors = True
continue
proper_org_name = drupal.get_organisation_name(org_id)
parent_department_id = drupal.get_department_from_organisation(
org_id)
orgs[org_id] = {
'name': proper_org_name,
'parent_department_id': parent_department_id
}
f = open(cls.lots_of_orgs_filepath, 'w')
try:
f.write(json.dumps(orgs))
finally:
f.close()
if has_errors:
print 'Finished with ERRORS'
sys.exit(1)
else:
print 'Finished with SUCCESS'
def test_04_am_following_user_missing_apikey(self):
params = json.dumps({'id': self.annafan.id})
response = self.app.post('/api/action/am_following_user',
params=params,
status=403).json
assert response['success'] == False
assert response['error']['message'] == 'Access denied'
def test_create_package(self):
test_pkg = self.get_package_fixture('test1')
offset = '/api/rest/package'
postparams = '%s=1' % json.dumps(test_pkg)
result = self.app.post(offset, postparams, status=[201], extra_environ=self.extra_environ_sysadmin)
# check returned dict is correct
res = json.loads(result.body)
assert_equal(res['name'], test_pkg['name'])
assert res['id']
assert_equal(res['title'], test_pkg['title'])
assert_equal(res['license_id'], test_pkg['license_id'])
assert_equal(res['extras'].get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(res['resources'][0].get('description'), test_pkg['resources'][0]['description'])
assert_equal(set(res['tags']), set(test_pkg['tags']))
# check package was created ok
pkg = model.Package.by_name(test_pkg['name'])
pkg_dict = get_action('package_show')(self.context, {'id': test_pkg['name']})
assert_equal(pkg.name, test_pkg['name'])
assert_equal(pkg.title, test_pkg['title'])
assert_equal(pkg.extras.get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(pkg.resources[0].description, test_pkg['resources'][0]['description'])
assert_equal(set([tag['name'] for tag in pkg_dict['tags']]), set(test_pkg['tags']))
def _finish(self, status_int, response_data=None,
content_type='text'):
'''When a controller method has completed, call this method
to prepare the response.
@return response message - return this value from the controller
method
e.g. return self._finish(404, 'Package not found')
'''
assert(isinstance(status_int, int))
response.status_int = status_int
response_msg = ''
if response_data is not None:
response.headers['Content-Type'] = CONTENT_TYPES[content_type]
if content_type == 'json':
response_msg = json.dumps(response_data)
else:
response_msg = response_data
# Support "JSONP" callback.
if status_int==200 and request.params.has_key('callback') and \
(request.method == 'GET' or \
c.logic_function and request.method == 'POST'):
# escape callback to remove '<', '&', '>' chars
callback = cgi.escape(request.params['callback'])
response_msg = self._wrap_jsonp(callback, response_msg)
return response_msg
def history_ajax(self, id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,}
data_dict = {'id':id}
try:
pkg_revisions = get_action('package_revision_list')(context, data_dict)
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Package not found'))
data = []
approved = False
for num, revision in enumerate(pkg_revisions):
if not approved and revision['approved_timestamp']:
current_approved, approved = True, True
else:
current_approved = False
data.append({'revision_id': revision['id'],
'message': revision['message'],
'timestamp': revision['timestamp'],
'author': revision['author'],
'approved': bool(revision['approved_timestamp']),
'current_approved': current_approved})
response.headers['Content-Type'] = 'application/json;charset=utf-8'
return json.dumps(data)
def resource_read(self, id, resource_id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
c.resource = get_action('resource_show')(context, {'id': resource_id})
c.package = get_action('package_show')(context, {'id': id})
# required for nav menu
c.pkg = context['package']
c.resource_json = json.dumps(c.resource)
c.pkg_dict = c.package
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
# get package license info
license_id = c.package.get('license_id')
try:
c.package['isopen'] = model.Package.get_license_register()[license_id].isopen()
except KeyError:
c.package['isopen'] = False
c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'),
qualified=True)
c.related_count = len(c.pkg.related)
c.num_followers = get_action('dataset_follower_count')(context,
{'id':c.pkg.id})
self._setup_follow_button(context)
return render('package/resource_read.html')
def before_index(self, pkg_dict):
'''Adds the fulltext of a package to the dict what
will be given to the solr for indexing.
@param pkg_dict: flattened dict (except for multli-valued fields such as tags)
containing all the terms which will be sent to the indexer
@return: modified package dict
'''
if pkg_dict and pkg_dict.has_key('extras_full_text_search'):
del pkg_dict['extras_full_text_search']
data_dict = json.loads(pkg_dict['data_dict'])
fulltext = [
x for x in data_dict['extras'] if 'full_text_search' in x['key']
]
if len(fulltext) > 0:
extras = [
x for x in data_dict['extras']
if not 'full_text_search' in x['key']
]
data_dict['extras'] = extras
pkg_dict['fulltext'] = fulltext[0]['value']
else:
fulltext_dict = _get_fulltext(pkg_dict['id'])
if fulltext_dict:
pkg_dict['fulltext'] = fulltext_dict.text
pkg_dict['data_dict'] = json.dumps(data_dict)
return pkg_dict
def before_index(self, pkg_dict):
'''Adds the fulltext of a package to the dict what
will be given to the solr for indexing.
@param pkg_dict: flattened dict (except for multli-valued fields such as tags)
containing all the terms which will be sent to the indexer
@return: modified package dict
'''
if pkg_dict and pkg_dict.has_key('extras_full_text_search'):
del pkg_dict['extras_full_text_search']
data_dict = json.loads(pkg_dict['data_dict'])
fulltext = [x for x in data_dict['extras'] if 'full_text_search' in x['key']]
if len(fulltext) > 0:
extras = [x for x in data_dict['extras'] if not 'full_text_search' in x['key']]
data_dict['extras'] = extras
pkg_dict['fulltext'] = fulltext[0]['value']
else:
fulltext_dict = _get_fulltext(pkg_dict['id'])
if fulltext_dict:
pkg_dict['fulltext'] = fulltext_dict.text
pkg_dict['data_dict'] = json.dumps(data_dict)
return pkg_dict
def edit(self, id, data=None, errors=None, error_summary=None):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,
'save': 'save' in request.params,
'moderated': config.get('moderated'),
'pending': True,
'schema': self._form_to_db_schema()
}
if context['save'] and not data:
return self._save_edit(id, context)
try:
old_data = get_action('package_show')(context, {'id': id})
schema = self._db_to_form_schema()
if schema and not data:
old_data, errors = validate(old_data, schema, context=context)
data = data or old_data
# Merge all elements for the complete package dictionary
c.pkg_dict = dict(old_data.items() + data.items())
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Package not found'))
c.pkg = context.get("package")
c.pkg_json = json.dumps(data)
try:
check_access('package_update', context)
except NotAuthorized, e:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
def edit(self, id, data=None, errors=None, error_summary=None):
package_type = self._get_package_type(id)
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,
'save': 'save' in request.params,
'moderated': config.get('moderated'),
'pending': True,
}
if context['save'] and not data:
return self._save_edit(id, context)
try:
c.pkg_dict = get_action('package_show')(context, {'id': id})
context['for_edit'] = True
old_data = get_action('package_show')(context, {'id': id})
# old data is from the database and data is passed from the
# user if there is a validation error. Use users data if there.
data = data or old_data
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Dataset not found'))
c.pkg = context.get("package")
c.resources_json = json.dumps(data.get('resources', []))
try:
check_access('package_update', context)
except NotAuthorized, e:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
def resource_read(self, id, resource_id):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
c.resource = get_action('resource_show')(context, {
'id': resource_id
})
c.package = get_action('package_show')(context, {'id': id})
# required for nav menu
c.pkg = context['package']
c.resource_json = json.dumps(c.resource)
c.pkg_dict = c.package
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
# get package license info
license_id = c.package.get('license_id')
try:
c.package['isopen'] = model.Package.get_license_register(
)[license_id].isopen()
except KeyError:
c.package['isopen'] = False
return render('package/resource_read.html')
def edit(self, id, data=None, errors=None, error_summary=None):
package_type = self._get_package_type(id)
context = {
"model": model,
"session": model.Session,
"user": c.user or c.author,
"extras_as_string": True,
"save": "save" in request.params,
"moderated": config.get("moderated"),
"pending": True,
}
if context["save"] and not data:
return self._save_edit(id, context)
try:
c.pkg_dict = get_action("package_show")(context, {"id": id})
context["for_edit"] = True
old_data = get_action("package_show")(context, {"id": id})
# old data is from the database and data is passed from the
# user if there is a validation error. Use users data if there.
data = data or old_data
except NotAuthorized:
abort(401, _("Unauthorized to read package %s") % "")
except NotFound:
abort(404, _("Dataset not found"))
c.pkg = context.get("package")
c.resources_json = json.dumps(data.get("resources", []))
try:
check_access("package_update", context)
except NotAuthorized, e:
abort(401, _("User %r not authorized to edit %s") % (c.user, id))
def _removens(content):
newstring = json.dumps(content)
newstring = re.sub("{http:[^}]+}","",newstring)
return newstring
def fetch_stage(self, harvest_object):
doc = html.parse(harvest_object.content)
package_dict = {'extras': {'harvest_dataset_url': harvest_object.content},
'resources': []}
package_dict['title'] = doc.findtext('//title').split(' | ')[0]
if not doc.find('//table[@class="BDE-table-frame vie-ogd-table"]'):
return False
for meta in doc.findall("//meta"):
key = meta.get('name')
value = meta.get('content')
if key is None or value is None:
continue
if key == 'DC.Creator':
package_dict['author'] = value
elif key == 'DC.date.created':
package_dict['metadata_created'] = value
elif key == 'DC.date.modified':
package_dict['metadata_modified'] = value
elif key == 'keywords':
package_dict['tags'] = value.split(',')
for row in doc.findall('//table[@class="BDE-table-frame vie-ogd-table"]//tr'):
key = row.find('th/p').text
elem = row.find('td')
if key == 'Beschreibung':
package_dict['notes'] = elem.xpath("string()")
elif key == 'Bezugsebene':
package_dict['extras']['geographic_coverage'] = elem.xpath("string()")
elif key == 'Zeitraum':
package_dict['extras']['temporal_coverage'] = elem.xpath("string()")
elif key == 'Aktualisierung':
package_dict['extras']['temporal_granularity'] = elem.xpath("string()")
elif key == 'Kategorien':
categories = elem.xpath("string()").split(',')
package_dict['extras']['categories'] = [c.strip() for c in categories]
elif key == 'Typ':
package_dict['extras']['type'] = elem.xpath("string()")
elif key == u'Attribute':
elem.tag = 'span'
package_dict['extras']['attributes'] = etree.tostring(elem)
elif key == u'Datenqualität':
package_dict['extras']['data_quality'] = elem.xpath("string()")
elif key == 'Kontakt':
package_dict['maintainer'] = elem.xpath("string()")
elif key == 'Lizenz':
if 'by/3.0/at/deed.de' in elem.findall('.//a')[0].get('href'):
package_dict['license_id'] = 'cc-by'
elif key == 'Datensatz':
for li in elem.findall('.//li'):
link = li.find('.//a').get('href')
if li.find('.//abbr') is not None:
res = {'description': li.xpath('string()'),
'url': link,
'format': li.find('.//abbr').text}
package_dict['resources'].append(res)
else:
package_dict['url'] = link
harvest_object.content = json.dumps(package_dict)
harvest_object.save()
return True
def generate(cls, xmlrpc_settings):
drupal = DrupalClient(xmlrpc_settings)
orgs = {}
has_errors = False
orgs_to_lookup = set()
orgs_to_lookup.add('Northern Ireland Executive')
for org_name in orgs_to_lookup:
org_name = canonise_organisation_name(org_name)
org_id = drupal.match_organisation(org_name)
if org_id == False:
log.error('Could not find organisation %r', org_name)
has_errors = True
continue
proper_org_name = drupal.get_organisation_name(org_id)
parent_department_id = drupal.get_department_from_organisation(org_id)
orgs[org_id] = {'name': proper_org_name,
'parent_department_id': parent_department_id}
f = open(cls.lots_of_orgs_filepath, 'w')
try:
f.write(json.dumps(orgs))
finally:
f.close()
if has_errors:
print 'Finished with ERRORS'
sys.exit(1)
else:
print 'Finished with SUCCESS'
def migrate1(self):
from ckan import model
from ckan.lib.helpers import json
q_status = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='status')
print '* %s with "status" will be deleted e.g. %s' % (q_status.count(),
q_status.first())
q_failures = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='openness_score_failure_count')
print '* %s with openness_score_failure_count to be deleted e.g.\n%s'\
% (q_failures.count(), q_failures.first())
q_score = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='openness_score')
print '* %s with openness_score to migrate e.g.\n%s' % \
(q_score.count(), q_score.first())
q_reason = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='openness_score_reason')
print '* %s with openness_score_reason to migrate e.g.\n%s' % \
(q_reason.count(), q_reason.first())
raw_input('Press Enter to continue')
q_status.delete()
model.Session.commit()
print '..."status" deleted'
q_failures.delete()
model.Session.commit()
print '..."openness_score_failure_count" deleted'
for task_status in q_score:
reason_task_status = q_reason \
.filter_by(entity_id=task_status.entity_id) \
.first()
if reason_task_status:
reason = reason_task_status.value
reason_task_status.delete()
else:
reason = None
task_status.key = 'status'
task_status.error = json.dumps({
'reason': reason,
'format': None,
'is_broken': None,
})
model.Session.commit()
print '..."openness_score" and "openness_score_reason" migrated'
count = q_reason.count()
q_reason.delete()
model.Session.commit()
print '... %i remaining "openness_score_reason" deleted' % count
model.Session.flush()
model.Session.remove()
print 'Migration succeeded'
def migrate1(self):
from ckan import model
from ckan.lib.helpers import json
q_status = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='status')
print '* %s with "status" will be deleted e.g. %s' % (q_status.count(),
q_status.first())
q_failures = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='openness_score_failure_count')
print '* %s with openness_score_failure_count to be deleted e.g.\n%s'\
% (q_failures.count(), q_failures.first())
q_score = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='openness_score')
print '* %s with openness_score to migrate e.g.\n%s' % \
(q_score.count(), q_score.first())
q_reason = model.Session.query(model.TaskStatus) \
.filter_by(task_type='qa') \
.filter_by(key='openness_score_reason')
print '* %s with openness_score_reason to migrate e.g.\n%s' % \
(q_reason.count(), q_reason.first())
raw_input('Press Enter to continue')
q_status.delete()
model.Session.commit()
print '..."status" deleted'
q_failures.delete()
model.Session.commit()
print '..."openness_score_failure_count" deleted'
for task_status in q_score:
reason_task_status = q_reason \
.filter_by(entity_id=task_status.entity_id) \
.first()
if reason_task_status:
reason = reason_task_status.value
reason_task_status.delete()
else:
reason = None
task_status.key = 'status'
task_status.error = json.dumps({
'reason': reason,
'format': None,
'is_broken': None,
})
model.Session.commit()
print '..."openness_score" and "openness_score_reason" migrated'
count = q_reason.count()
q_reason.delete()
model.Session.commit()
print '... %i remaining "openness_score_reason" deleted' % count
model.Session.flush()
model.Session.remove()
print 'Migration succeeded'
def fetch_stage(self, harvest_object):
doc = html.parse(harvest_object.content)
package_dict = {'extras': {}, 'resources': [], 'tags': []}
package_dict['title'] = doc.findtext('//h2[@class="itemTitle"]').strip()
package_dict['notes'] = doc.find('//div[@class="itemFullText"]').xpath('string()').strip()
source = doc.find('//div[@class="itemFonte"]/a')
if source is not None:
package_dict['url'] = source.get('href')
package_dict['author'] = doc.find('//div[@class="itemFonte"]').xpath('string()')\
.replace('Fonte:', '').strip()
package_dict['extras']['harvest_dataset_url'] = harvest_object.content
file_type = ''
for block in doc.findall('//div[@class="itemBlock"]'):
name = block.findtext('h3')
if 'Tipo di file' in name:
file_type = block.find('p').xpath('string()')
elif 'Scala' in name:
package_dict['extras']['scale'] = \
block.find('p').xpath('string()')
elif 'Sistema di riferimento' in name:
package_dict['extras']['reference_system'] = \
block.find('p').xpath('string()')
elif 'Frequenza di aggiornamento' in name:
package_dict['extras']['temporal_granularity'] = \
block.find('p').xpath('string()').strip()
elif 'Data aggiornamento della scheda' in name:
package_dict['metadata_modified'] = \
block.find('p').xpath('string()').strip()
elif 'Data aggiornamento del dato' in name:
package_dict['metadata_created'] = \
block.find('p').xpath('string()').strip()
elif 'Ente proprietario' in name:
if block.find('h3').tail:
package_dict['maintainer'] = \
block.find('h3').tail.strip()
elif 'Tag' in name:
for a in block.findall('.//a'):
package_dict['tags'].append(a.text)
elif 'Argomenti' in name:
package_dict['extras']['categories'] = []
for span in block.findall('span'):
if span.tail:
package_dict['extras']['categories'].append(span.tail)
downloadForm = doc.find("//form[@id='downloadForm']")
if downloadForm:
_dl = urljoin(self.INDEX_URL, downloadForm.get('action'))
fh = urlopen(_dl)
package_dict['resources'].append({
'url': fh.url,
'format': file_type,
'description': os.path.basename(fh.url)
})
fh.close()
#from pprint import pprint
#pprint(package_dict)
#package_dict['license_id'] = 'odc-odbl'
harvest_object.content = json.dumps(package_dict)
harvest_object.save()
return True
def _delete_vocabulary(self, vocab_id, user=None):
if user:
extra_environ = {'Authorization': str(user.apikey)}
else:
extra_environ = None
params = {'id': vocab_id}
response = self._post('/api/action/vocabulary_delete',
params=params,
extra_environ=extra_environ)
# Check the values of the response.
assert response['success'] == True
assert response['result'] is None
response['result']
# Get the list of vocabularies.
response = self._post('/api/action/vocabulary_list')
assert response['success'] == True
assert response['result']
# Check that the vocabulary we deleted is not in the list.
assert vocab_id not in [vocab['id'] for vocab in response['result']]
# Check that the deleted vocabulary can no longer be retrieved.
response = self.app.post(
'/api/action/vocabulary_show',
params=json.dumps(params),
extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
status=404)
assert response.json['success'] == False
def _delete_vocabulary(self, vocab_id, user=None):
if user:
extra_environ = {'Authorization' : str(user.apikey)}
else:
extra_environ = None
params = {'id': vocab_id}
response = self._post('/api/action/vocabulary_delete', params=params,
extra_environ=extra_environ)
# Check the values of the response.
assert response['success'] == True
assert response['result'] is None
response['result']
# Get the list of vocabularies.
response = self._post('/api/action/vocabulary_list')
assert response['success'] == True
assert response['result']
# Check that the vocabulary we deleted is not in the list.
assert vocab_id not in [vocab['id'] for vocab in response['result']]
# Check that the deleted vocabulary can no longer be retrieved.
response = self.app.post('/api/action/vocabulary_show',
params=json.dumps(params),
extra_environ = {'Authorization':
str(self.sysadmin_user.apikey)},
status=404)
assert response.json['success'] == False
def gather_stage(self, harvest_job):
log.debug('In ZhstatHarvester gather_stage')
ids = []
parser = etree.XMLParser(encoding='utf-8')
for dataset in etree.fromstring(self._fetch_metadata(), parser=parser):
# Get the german data if one is available,
# otherwise get the first one
base_datas = dataset.xpath("data[@xml:lang='de']")
if len(base_datas) != 0:
base_data = base_datas[0]
else:
base_data = dataset.find('data')
metadata = self._generate_metadata(base_data, dataset)
if metadata:
obj = HarvestObject(guid=dataset.get('id'),
job=harvest_job,
content=json.dumps(metadata))
obj.save()
log.debug('adding ' + dataset.get('id') + ' to the queue')
ids.append(obj.id)
else:
log.debug(
'Skipping %s since no resources or groups are available' %
dataset.get('id'))
return ids
def test_create_package(self):
test_pkg = self.get_package_fixture('test1')
offset = '/api/rest/package'
postparams = '%s=1' % json.dumps(test_pkg)
result = self.app.post(offset, postparams, status=[201], extra_environ=self.extra_environ_sysadmin)
# check returned dict is correct
res = json.loads(result.body)
assert_equal(res['name'], test_pkg['name'])
assert res['id']
assert_equal(res['title'], test_pkg['title'])
assert_equal(res['license_id'], test_pkg['license_id'])
assert_equal(res['extras'].get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(res['resources'][0].get('description'), test_pkg['resources'][0]['description'])
assert_equal(set(res['tags']), set(test_pkg['tags']))
# check package was created ok
pkg = model.Package.by_name(test_pkg['name'])
pkg_dict = get_action('package_show')(self.context, {'id': test_pkg['name']})
assert_equal(pkg.name, test_pkg['name'])
assert_equal(pkg.title, test_pkg['title'])
assert_equal(pkg.extras.get('temporal_coverage-to'), test_pkg['extras']['temporal_coverage-to'])
assert_equal(pkg.resources[0].description, test_pkg['resources'][0]['description'])
assert_equal(set([tag['name'] for tag in pkg_dict['tags']]), set(test_pkg['tags']))
def history_ajax(self, id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,}
data_dict = {'id':id}
try:
pkg_revisions = get_action('package_revision_list')(context, data_dict)
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Dataset not found'))
data = []
approved = False
for num, revision in enumerate(pkg_revisions):
if not approved and revision['approved_timestamp']:
current_approved, approved = True, True
else:
current_approved = False
data.append({'revision_id': revision['id'],
'message': revision['message'],
'timestamp': revision['timestamp'],
'author': revision['author'],
'approved': bool(revision['approved_timestamp']),
'current_approved': current_approved})
response.headers['Content-Type'] = 'application/json;charset=utf-8'
return json.dumps(data)
def data(self):
# Get the Europe dataset
rootdir = get_root_dir()
data_file = os.path.join(rootdir, 'ckanext', 'offenedaten', 'data', 'eu.json')
f = open(data_file, 'r')
o = json.load(f)
# Get the package count by country
q = Session.query(
distinct(PackageExtra.value),
func.count(PackageExtra.value)
).\
filter(PackageExtra.key == u'eu_country').\
group_by(PackageExtra.value)
values = dict(q.all())
# Set the package count for each country
for ft in o['features']:
code = ft['properties']['NUTS']
ft['properties']['packages'] = (values.get(code, 0))
response.content_type = 'application/json'
response.pragma = None
response.cache_control = 'public; max-age: 3600'
response.cache_expires(seconds=3600)
return json.dumps(o)
def edit(self, id, data=None, errors=None, error_summary=None):
package_type = self._get_package_type(id)
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params,
'moderated': config.get('moderated'),
'pending': True,}
if context['save'] and not data:
return self._save_edit(id, context)
try:
old_data = get_action('package_show')(context, {'id':id})
schema = self._db_to_form_schema(package_type=package_type)
if schema and not data:
old_data, errors = validate(old_data, schema, context=context)
data = data or old_data
# Merge all elements for the complete package dictionary
c.pkg_dict = dict(old_data.items() + data.items())
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Dataset not found'))
c.pkg = context.get("package")
c.pkg_json = json.dumps(data)
try:
check_access('package_update',context)
except NotAuthorized, e:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
def test_01_unfollow_dataset_missing_object_id(self):
params = json.dumps({})
extra_environ = {'Authorization': str(self.annafan.apikey),}
response = self.app.post('/api/action/unfollow_dataset',
params=params, extra_environ=extra_environ, status=409).json
assert response['success'] == False
assert response['error']['id'] == ['Missing value']
def test_04_dataset_follower_list_bad_id(self):
for object_id in ('bad id', ' ', 3, 35.7, 'xxx', ''):
params = json.dumps({'id': object_id})
response = self.app.post('/api/action/dataset_follower_list',
params=params, status=409).json
assert response['success'] is False
assert response['error']['id']
def validate_config(self, config):
'''
Harvesters can provide this method to validate the configuration
entered in the form. It should return a single string, which will be
stored in the database. Exceptions raised will be shown in the form's
error messages.
:param harvest_object_id: Config string coming from the form
:returns: A string with the validated configuration options
'''
if not config:
return config
try:
config_obj = json.loads(config)
# This method is used to check if 'element' is in config_obj
# and that it is defined as either a string or a list of strings.
def check_if_element_is_string_or_list_in_config_obj(element):
if element in config_obj:
if not isinstance(config_obj[element], list):
if not isinstance(config_obj[element], basestring):
raise ValueError(
'%s must be a string '
'or a list of strings, %s is neither' %
(element, config_obj[element]))
else:
for item in config_obj[element]:
if not isinstance(item, basestring):
raise ValueError(
'%s must be a string '
'or a list of strings, %s is neither' %
(item, config_obj[element][item]))
# Check if 'filter' is a string or a list of strings if it is defined
check_if_element_is_string_or_list_in_config_obj('themes')
check_if_element_is_string_or_list_in_config_obj('organizations')
check_if_element_is_string_or_list_in_config_obj('text')
check_if_element_is_string_or_list_in_config_obj('title')
check_if_element_is_string_or_list_in_config_obj('uuid')
check_if_element_is_string_or_list_in_config_obj('datatypes')
check_if_element_is_string_or_list_in_config_obj('default_tags')
# Check if 'create_orgs' is set to 'create' if it is defined
if 'create_orgs' in config_obj and not isinstance(
config_obj['create_orgs'], bool):
raise ValueError(
'create_orgs must be a boolean, either True or False')
# Check if 'force_all' is a boolean value
if 'force_all' in config_obj and not isinstance(
config_obj['force_all'], bool):
raise ValueError(
'force_all must be a boolean, either True or False')
config = json.dumps(config_obj)
except ValueError, e:
raise e
def check_dataset_read_page(self,
response,
name=None,
title=None,
tag_string=None,
group=None,
cmap_geographical_level=None,
cmap_data_family=None,
cmap_data_category=None,
cmap_data_subcategory=None,
cmap_data_field=None):
'''Check the contents of a dataset read page.
Given the response from a dataset read page, tests that the page
contains the right contents such as the link to the group's website,
the group's logo, etc.
'''
# Test the the values for ckanext-cmap's custom dataset metadata fields
# show on the page.
for metadata_field in (cmap_geographical_level, cmap_data_family,
cmap_data_category, cmap_data_subcategory,
cmap_data_field):
if metadata_field is not None:
assert metadata_field in response
# Test that the logo of the dataset's group shows on the dataset's
# page, is linked to the group's website, and has the group's title as
# alt text.
if group is not None:
# Get the group from the CKAN API.
params = {'id': group}
api_response = self.app.post('/api/action/group_show',
params=json.dumps(params)).json
assert api_response['success'] is True
image_url = api_response['result'].get('image_url', None)
website_url = api_response['result'].get('website_url', None)
# Workaround what seems to be a bug in the CKAN API.
if website_url.startswith('"'):
website_url = website_url[1:]
if website_url.endswith('"'):
website_url = website_url[:-1]
group_title = api_response['result'].get('title', None)
if image_url is not None:
soup = BeautifulSoup(response.body)
img = soup.find('img', src=image_url)
assert img, ("Organization's logo should appear on pages of "
"organization's datasets")
assert img.get('alt', None) == group_title, (
"Organization's "
"logo should have organization's title as alt text")
if website_url is not None:
assert img.find_parents('a', href=website_url), (
"Organization's logo should be hyperlinked to "
"organization's website on organization's dataset's "
"pages")
def call_action_api(app, action, apikey=None, status=200, **kwargs):
'''POST an HTTP request to the CKAN API and return the result.
Any additional keyword arguments that you pass to this function as **kwargs
are posted as params to the API.
Usage:
package_dict = post(app, 'package_create', apikey=apikey,
name='my_package')
assert package_dict['name'] == 'my_package'
num_followers = post(app, 'user_follower_count', id='annafan')
If you are expecting an error from the API and want to check the contents
of the error dict, you have to use the status param otherwise an exception
will be raised:
error_dict = post(app, 'group_activity_list', status=403,
id='invalid_id')
assert error_dict['message'] == 'Access Denied'
:param app: the test app to post to
:type app: paste.fixture.TestApp
:param action: the action to post to, e.g. 'package_create'
:type action: string
:param apikey: the API key to put in the Authorization header of the post
(optional, default: None)
:type apikey: string
:param status: the HTTP status code expected in the response from the CKAN
API, e.g. 403, if a different status code is received an exception will
be raised (optional, default: 200)
:type status: int
:param **kwargs: any other keyword arguments passed to this function will
be posted to the API as params
:raises paste.fixture.AppError: if the HTTP status code of the response
from the CKAN API is different from the status param passed to this
function
:returns: the 'result' or 'error' dictionary from the CKAN API response
:rtype: dictionary
'''
params = json.dumps(kwargs)
response = app.post('/api/action/{0}'.format(action),
params=params,
extra_environ={'Authorization': str(apikey)},
status=status)
if status in (200, ):
assert response.json['success'] is True
return response.json['result']
else:
assert response.json['success'] is False
return response.json['error']