def details(short_name):
app = app_by_shortname(short_name)
try:
require.app.read(app)
template = '/applications/app.html'
except HTTPException:
if app.hidden:
raise abort(403)
else:
raise
title = app_title(app, None)
template_args = {"app": app, "title": title}
try:
if current_app.config.get('CKAN_URL'):
template_args['ckan_name'] = current_app.config.get('CKAN_NAME')
ckan = Ckan(url=current_app.config['CKAN_URL'])
pkg, e = ckan.package_exists(name=short_name)
if e:
raise e
if pkg:
template_args['ckan_pkg_url'] = (
"%s/dataset/%s" % (current_app.config['CKAN_URL'], short_name))
template_args['ckan_pkg'] = pkg
except requests.exceptions.ConnectionError:
current_app.logger.error("CKAN server down or there is a typo in the URL")
except Exception as e:
current_app.logger.error(e)
return render_template(template, **template_args)
def respond_ckan(ty):
# First check if there is a package (dataset) in CKAN
tables = {"task": model.Task, "task_run": model.TaskRun}
msg_1 = gettext("Data exported to ")
msg = msg_1 + "%s ..." % current_app.config['CKAN_URL']
ckan = Ckan(url=current_app.config['CKAN_URL'],
api_key=current_user.ckan_api)
app_url = url_for('.details', short_name=app.short_name, _external=True)
try:
package, e = ckan.package_exists(name=app.short_name)
if e:
raise e
if package:
# Update the package
owner = User.query.get(app.owner_id)
package = ckan.package_update(app=app, user=owner, url=app_url,
resources=package['resources'])
ckan.package = package
resource_found = False
# print len(package['resources'])
for r in package['resources']:
if r['name'] == ty:
ckan.datastore_delete(name=ty, resource_id=r['id'])
ckan.datastore_create(name=ty, resource_id=r['id'])
ckan.datastore_upsert(name=ty,
records=gen_json(tables[ty]),
resource_id=r['id'])
resource_found = True
break
if not resource_found:
create_ckan_datastore(ckan, ty, package['id'])
else:
owner = User.query.get(app.owner_id)
package = ckan.package_create(app=app, user=owner, url=app_url)
create_ckan_datastore(ckan, ty, package['id'])
#new_resource = ckan.resource_create(name=ty,
# package_id=package['id'])
#ckan.datastore_create(name=ty,
# resource_id=new_resource['result']['id'])
#ckan.datastore_upsert(name=ty,
# records=gen_json(tables[ty]),
# resource_id=new_resource['result']['id'])
flash(msg, 'success')
return respond()
except requests.exceptions.ConnectionError:
msg = "CKAN server seems to be down, try again layer or contact the CKAN admins"
current_app.logger.error(msg)
flash(msg, 'danger')
except Exception as inst:
if len(inst.args) == 3:
t, msg, status_code = inst.args
msg = ("Error: %s with status code: %s" % (t, status_code))
else: # pragma: no cover
msg = ("Error: %s" % inst.args[0])
current_app.logger.error(msg)
flash(msg, 'danger')
finally:
return respond()
def respond_ckan(ty):
# First check if there is a package (dataset) in CKAN
tables = {"task": model.Task, "task_run": model.TaskRun}
msg_1 = lazy_gettext("Data exported to ")
msg = msg_1 + "%s ..." % current_app.config['CKAN_URL']
ckan = Ckan(url=current_app.config['CKAN_URL'],
api_key=current_user.ckan_api)
app_url = url_for('.details',
short_name=app.short_name,
_external=True)
try:
package = ckan.package_exists(name=app.short_name)
if package:
# Update the package
ckan.package_update(app=app, user=app.owner, url=app_url)
if len(package['resources']) == 0:
resources = create_ckan_datastores(ckan)
ckan.datastore_upsert(name=ty,
records=gen_json(tables[ty]),
resource_id=resources[ty]['id'])
flash(msg, 'success')
return render_template('/applications/export.html',
title=title,
app=app)
else:
ckan.datastore_delete(name=ty)
ckan.datastore_create(name=ty)
ckan.datastore_upsert(name=ty,
records=gen_json(tables[ty]))
flash(msg, 'success')
return render_template('/applications/export.html',
title=title,
app=app)
else:
ckan.package_create(app=app,
user=app.owner,
url=app_url,
tags=current_app.config['BRAND'])
resources = create_ckan_datastores(ckan)
ckan.datastore_upsert(name=ty,
records=gen_json(tables[ty]),
resource_id=resources[ty]['id'])
flash(msg, 'success')
return render_template('/applications/export.html',
title=title,
app=app)
except Exception as inst:
print inst
if len(inst.args) == 3:
type, msg, status_code = inst.args
msg = ("Error: %s with status code: %s" % (type, status_code))
else:
msg = ("Error: %s" % inst.args[0])
flash(msg, 'danger')
return render_template('/applications/export.html',
title=title,
app=app)
def respond_ckan(ty):
# First check if there is a package (dataset) in CKAN
tables = {"task": model.Task, "task_run": model.TaskRun}
msg_1 = lazy_gettext("Data exported to ")
msg = msg_1 + "%s ..." % current_app.config['CKAN_URL']
ckan = Ckan(url=current_app.config['CKAN_URL'],
api_key=current_user.ckan_api)
app_url = url_for('.details',
short_name=app.short_name,
_external=True)
try:
package, e = ckan.package_exists(name=app.short_name)
if e:
raise e
if package:
# Update the package
ckan.package_update(app=app, user=app.owner, url=app_url)
if len(package['resources']) == 0:
resources = create_ckan_datastores(ckan)
ckan.datastore_upsert(name=ty,
records=gen_json(tables[ty]),
resource_id=resources[ty]['id'])
else:
ckan.datastore_delete(name=ty)
ckan.datastore_create(name=ty)
ckan.datastore_upsert(name=ty,
records=gen_json(tables[ty]))
else:
ckan.package_create(app=app,
user=app.owner,
url=app_url,
tags=current_app.config['BRAND'])
resources = create_ckan_datastores(ckan)
ckan.datastore_upsert(name=ty,
records=gen_json(tables[ty]),
resource_id=resources[ty]['id'])
flash(msg, 'success')
return respond()
except requests.exceptions.ConnectionError:
msg = "CKAN server seems to be down, try again layer or contact the CKAN admins"
current_app.logger.error(msg)
flash(msg, 'danger')
except Exception as inst:
if len(inst.args) == 3:
type, msg, status_code = inst.args
msg = ("Error: %s with status code: %s" % (type, status_code))
else:
msg = ("Error: %s" % inst.args[0])
current_app.logger.error(msg)
flash(msg, 'danger')
finally:
return respond()
def details(short_name):
app = app_by_shortname(short_name)
try:
require.app.read(app)
require.app.update(app)
template = '/applications/actions.html'
except HTTPException:
if app.hidden:
app = None
template = '/applications/app.html'
title = app_title(app, None)
template_args = {"app": app, "title": title}
if current_app.config.get('CKAN_URL'):
template_args['ckan_name'] = current_app.config.get('CKAN_NAME')
ckan = Ckan(url=current_app.config['CKAN_URL'])
pkg = ckan.package_exists(name=short_name)
if pkg:
template_args['ckan_pkg_url'] = (
"%s/dataset/%s" % (current_app.config['CKAN_URL'], short_name))
template_args['ckan_pkg'] = pkg
return render_template(template, **template_args)
class TestCkanModule(Test, object):
ckan = Ckan(url="http://datahub.io", api_key="fake-api-key")
task_resource_id = "0dde48c7-a0e9-445f-bc84-6365ec057450"
task_run_resource_id = "a49448dc-228a-4d54-a697-ba02c50e0143"
package_id = "4bcf844c-3ad0-4203-8418-32e7d7c4ce96"
pkg_json_not_found = {
"help": "Return ...",
"success": False,
"error": {
"message": "Not found",
"__type": "Not Found Error"
}
}
pkg_json_found = {
"help": "Return the metadata of a dataset ...",
"success": True,
"result": {
"license_title":
"",
"maintainer":
"",
"relationships_as_object": [],
"maintainer_email":
"",
"revision_timestamp":
"2013-04-11T11:45:52.689160",
"id":
package_id,
"metadata_created":
"2013-04-11T11:39:56.003541",
"metadata_modified":
"2013-04-12T10:50:45.132825",
"author":
"Daniel Lombrana Gonzalez",
"author_email":
"",
"state":
"deleted",
"version":
"",
"license_id":
"",
"type":
None,
"resources": [{
"resource_group_id": "f45c29ce-97f3-4b1f-b060-3306ffedb64b",
"cache_last_updated": None,
"revision_timestamp": "2013-04-12T10:50:41.635556",
"webstore_last_updated": None,
"id": task_resource_id,
"size": None,
"state": "active",
"last_modified": None,
"hash": "",
"description": "tasks",
"format": "",
"tracking_summary": {
"total": 0,
"recent": 0
},
"mimetype_inner": None,
"mimetype": None,
"cache_url": None,
"name": "task",
"created": "2013-04-12T05:50:41.776512",
"url": "http://*****:*****@patch('pybossa.ckan.requests.get')
def test_00_package_exists_returns_false(self, Mock):
"""Test CKAN get_resource_id works"""
html_request = FakeRequest(json.dumps(self.pkg_json_not_found), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
# Resource that exists
out, e = self.ckan.package_exists(name='not-found')
assert out is False, "It should return False as pkg does not exist"
# Handle error in CKAN server
Mock.return_value = self.server_error
try:
pkg, e = self.ckan.package_exists(name="something-goes-wrong")
if e:
raise e
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, msg
assert status_code == 500, "status_code should be 500"
assert type == "CKAN: the remote site failed! package_show failed"
# Now with a broken JSON item
Mock.return_value = FakeRequest("simpletext", 200,
{'content-type': 'text/html'})
out, e = self.ckan.package_exists(name='not-found')
assert out is False, "It should return False as pkg does not exist"
# Handle error in CKAN server
try:
pkg, e = self.ckan.package_exists(name="something-goes-wrong")
if e:
raise e
except Exception as out:
type, msg, status_code = out.args
assert status_code == 200, "status_code should be 200"
assert type == "CKAN: JSON not valid"
@patch('pybossa.ckan.requests.get')
def test_01_package_exists_returns_pkg(self, Mock):
"""Test CKAN get_resource_id works"""
html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
# Resource that exists
out, e = self.ckan.package_exists(name='urbanpark')
assert out is not False, "It should return a pkg"
err_msg = "The pkg id should be the same"
assert out['id'] == self.pkg_json_found['result']['id'], err_msg
@patch('pybossa.ckan.requests.get')
def test_02_get_resource_id(self, Mock):
"""Test CKAN get_resource_id works"""
html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
# Resource that exists
# Get the package
out, e = self.ckan.package_exists(name='urbanpark')
# Get the resource id for Task
out = self.ckan.get_resource_id(name='task')
err_msg = "It should return the task resource ID"
assert out == self.task_resource_id, err_msg
# Get the resource id for TaskRun
out = self.ckan.get_resource_id(name='task_run')
err_msg = "It should return the task_run resource ID"
assert out == self.task_run_resource_id, err_msg
# Get the resource id for a non existant resource
err_msg = "It should return false"
out = self.ckan.get_resource_id(name='non-existant')
assert out is False, err_msg
@patch('pybossa.ckan.requests.post')
def test_03_package_create(self, Mock):
"""Test CKAN package_create works"""
# It should return self.pkg_json_found with an empty Resources list
html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
# Resource that exists
project = Project(short_name='urbanpark', name='Urban Parks')
user = User(fullname='Daniel Lombrana Gonzalez')
out = self.ckan.package_create(project=project,
user=user,
url="http://something.com")
err_msg = "The package ID should be the same"
assert out['id'] == self.package_id, err_msg
# Check the exception
Mock.return_value = self.server_error
try:
self.ckan.package_create(project=project,
user=user,
url="http://something.com")
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! package_create failed" == type, type
@patch('pybossa.ckan.requests.post')
def test_05_resource_create(self, Mock):
"""Test CKAN resource_create works"""
pkg_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
{'content-type': 'application/json'})
rsrc_request = FakeRequest(
json.dumps(self.pkg_json_found['result']['resources'][0]), 200,
{'content-type': 'text/html'})
Mock.return_value = pkg_request
with self.flask_app.test_request_context('/'):
# Resource that exists
project = Project(short_name='urbanpark', name='Urban Parks')
user = User(fullname='Daniel Lombrana Gonzalez')
self.ckan.package_create(project=project,
user=user,
url="http://something.com")
Mock.return_value = rsrc_request
out = self.ckan.resource_create(name='task')
err_msg = "It should create the task resource"
assert out["id"] == self.task_resource_id, err_msg
Mock.return_value = self.server_error
try:
self.ckan.resource_create(name='something-goes-wrong')
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! resource_create failed" == type, type
@patch('pybossa.ckan.requests.post')
def test_05_datastore_create_without_resource_id(self, Mock):
"""Test CKAN datastore_create without resource_id works"""
html_request = FakeRequest(json.dumps(self.task_datastore), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
out = self.ckan.datastore_create(name='task', resource_id=None)
err_msg = "It should ref the task resource ID"
assert out['resource_id'] == self.task_resource_id, err_msg
# Check the error
Mock.return_value = self.server_error
try:
self.ckan.datastore_create(name='task',
resource_id=self.task_resource_id)
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, err_msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! datastore_create failed" == type, type
@patch('pybossa.ckan.requests.post')
def test_05_datastore_create(self, Mock):
"""Test CKAN datastore_create works"""
html_request = FakeRequest(json.dumps(self.task_datastore), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
out = self.ckan.datastore_create(name='task',
resource_id=self.task_resource_id)
err_msg = "It should ref the task resource ID"
assert out['resource_id'] == self.task_resource_id, err_msg
# Check the error
Mock.return_value = self.server_error
try:
self.ckan.datastore_create(name='task',
resource_id=self.task_resource_id)
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, err_msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! datastore_create failed" == type, type
@patch('pybossa.ckan.requests.post')
def test_06_datastore_upsert_without_resource_id(self, Mock):
"""Test CKAN datastore_upsert without resourece_id works"""
html_request = FakeRequest(json.dumps(self.task_upsert), 200,
{'content-type': 'application/json'})
record = dict(info=dict(foo="bar"))
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
out = self.ckan.datastore_upsert(name='task',
records=json.dumps([record]),
resource_id=None)
err_msg = "It should return True"
assert out is True, err_msg
# Check the error
Mock.return_value = self.server_error
try:
self.ckan.datastore_upsert(name='task',
records=json.dumps([record]),
resource_id=self.task_resource_id)
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! datastore_upsert failed" == type, type
@patch('pybossa.ckan.requests.post')
def test_06_datastore_upsert(self, Mock):
"""Test CKAN datastore_upsert works"""
html_request = FakeRequest(json.dumps(self.task_upsert), 200,
{'content-type': 'application/json'})
record = dict(info=dict(foo="bar"))
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
out = self.ckan.datastore_upsert(name='task',
records=json.dumps([record]),
resource_id=self.task_resource_id)
err_msg = "It should return True"
assert out is True, err_msg
# Check the error
Mock.return_value = self.server_error
try:
self.ckan.datastore_upsert(name='task',
records=json.dumps([record]),
resource_id=self.task_resource_id)
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! datastore_upsert failed" == type, type
@patch('pybossa.ckan.requests.post')
def test_07_datastore_delete(self, Mock):
"""Test CKAN datastore_delete works"""
html_request = FakeRequest(json.dumps({}), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
out = self.ckan.datastore_delete(name='task',
resource_id=self.task_resource_id)
err_msg = "It should return True"
assert out is True, err_msg
# Check the error
Mock.return_value = self.server_error
try:
self.ckan.datastore_delete(name='task',
resource_id=self.task_resource_id)
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! datastore_delete failed" == type, type
@patch('pybossa.ckan.requests.post')
def test_08_package_update(self, Mock):
"""Test CKAN package_update works"""
html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
{'content-type': 'application/json'})
Mock.return_value = html_request
with self.flask_app.test_request_context('/'):
# Resource that exists
project = Project(short_name='urbanpark', name='Urban Parks')
user = User(fullname='Daniel Lombrana Gonzalez')
out = self.ckan.package_update(
project=project,
user=user,
url="http://something.com",
resources=self.pkg_json_found['result']['resources'])
err_msg = "The package ID should be the same"
assert out['id'] == self.package_id, err_msg
# Check the exception
Mock.return_value = self.server_error
try:
self.ckan.package_update(
project=project,
user=user,
url="http://something.com",
resources=self.pkg_json_found['result']['resources'])
except Exception as out:
type, msg, status_code = out.args
assert "Server Error" in msg, msg
assert 500 == status_code, status_code
assert "CKAN: the remote site failed! package_update failed" == type, type