def details(short_name): app, n_tasks, n_task_runs, overall_progress, last_activity = app_by_shortname(short_name) try: require.app.read(app) template = '/applications/app.html' except HTTPException: if app.hidden: raise abort(403) else: raise title = app_title(app, None) template_args = {"app": app, "title": title, "n_tasks": n_tasks, "overall_progress": overall_progress, "last_activity": last_activity} try: if current_app.config.get('CKAN_URL'): template_args['ckan_name'] = current_app.config.get('CKAN_NAME') ckan = Ckan(url=current_app.config['CKAN_URL']) pkg, e = ckan.package_exists(name=short_name) if e: raise e if pkg: template_args['ckan_pkg_url'] = ( "%s/dataset/%s" % (current_app.config['CKAN_URL'], short_name)) template_args['ckan_pkg'] = pkg except requests.exceptions.ConnectionError: current_app.logger.error("CKAN server down or there is a typo in the URL") except Exception as e: current_app.logger.error(e) return render_template(template, **template_args)
def details(short_name): app = app_by_shortname(short_name) try: require.app.read(app) template = '/applications/app.html' except HTTPException: if app.hidden: raise abort(403) else: raise title = app_title(app, None) template_args = {"app": app, "title": title} try: if current_app.config.get('CKAN_URL'): template_args['ckan_name'] = current_app.config.get('CKAN_NAME') ckan = Ckan(url=current_app.config['CKAN_URL']) pkg, e = ckan.package_exists(name=short_name) if e: raise e if pkg: template_args['ckan_pkg_url'] = ( "%s/dataset/%s" % (current_app.config['CKAN_URL'], short_name)) template_args['ckan_pkg'] = pkg except requests.exceptions.ConnectionError: current_app.logger.error("CKAN server down or there is a typo in the URL") except Exception as e: current_app.logger.error(e) return render_template(template, **template_args)
def details(short_name): app = app_by_shortname(short_name) try: require.app.read(app) require.app.update(app) template = '/applications/actions.html' except HTTPException: if app.hidden: app = None template = '/applications/app.html' title = app_title(app, None) template_args = {"app": app, "title": title} if current_app.config.get('CKAN_URL'): template_args['ckan_name'] = current_app.config.get('CKAN_NAME') ckan = Ckan(url=current_app.config['CKAN_URL']) pkg = ckan.package_exists(name=short_name) if pkg: template_args['ckan_pkg_url'] = ( "%s/dataset/%s" % (current_app.config['CKAN_URL'], short_name)) template_args['ckan_pkg'] = pkg return render_template(template, **template_args)
class TestCkanModule(Test, object): ckan = Ckan(url="http://datahub.io", api_key="fake-api-key") task_resource_id = "0dde48c7-a0e9-445f-bc84-6365ec057450" task_run_resource_id = "a49448dc-228a-4d54-a697-ba02c50e0143" package_id = "4bcf844c-3ad0-4203-8418-32e7d7c4ce96" pkg_json_not_found = { "help": "Return ...", "success": False, "error": { "message": "Not found", "__type": "Not Found Error" } } pkg_json_found = { "help": "Return the metadata of a dataset ...", "success": True, "result": { "license_title": "", "maintainer": "", "relationships_as_object": [], "maintainer_email": "", "revision_timestamp": "2013-04-11T11:45:52.689160", "id": package_id, "metadata_created": "2013-04-11T11:39:56.003541", "metadata_modified": "2013-04-12T10:50:45.132825", "author": "Daniel Lombrana Gonzalez", "author_email": "", "state": "deleted", "version": "", "license_id": "", "type": None, "resources": [{ "resource_group_id": "f45c29ce-97f3-4b1f-b060-3306ffedb64b", "cache_last_updated": None, "revision_timestamp": "2013-04-12T10:50:41.635556", "webstore_last_updated": None, "id": task_resource_id, "size": None, "state": "active", "last_modified": None, "hash": "", "description": "tasks", "format": "", "tracking_summary": { "total": 0, "recent": 0 }, "mimetype_inner": None, "mimetype": None, "cache_url": None, "name": "task", "created": "2013-04-12T05:50:41.776512", "url": "http://*****:*****@patch('pybossa.ckan.requests.get') def test_00_package_exists_returns_false(self, Mock): """Test CKAN get_resource_id works""" html_request = FakeRequest(json.dumps(self.pkg_json_not_found), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): # Resource that exists out, e = self.ckan.package_exists(name='not-found') assert out is False, "It should return False as pkg does not exist" # Handle error in CKAN server Mock.return_value = self.server_error try: pkg, e = self.ckan.package_exists(name="something-goes-wrong") if e: raise e except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, msg assert status_code == 500, "status_code should be 500" assert type == "CKAN: the remote site failed! package_show failed" # Now with a broken JSON item Mock.return_value = FakeRequest("simpletext", 200, {'content-type': 'text/html'}) out, e = self.ckan.package_exists(name='not-found') assert out is False, "It should return False as pkg does not exist" # Handle error in CKAN server try: pkg, e = self.ckan.package_exists(name="something-goes-wrong") if e: raise e except Exception as out: type, msg, status_code = out.args assert status_code == 200, "status_code should be 200" assert type == "CKAN: JSON not valid" @patch('pybossa.ckan.requests.get') def test_01_package_exists_returns_pkg(self, Mock): """Test CKAN get_resource_id works""" html_request = FakeRequest(json.dumps(self.pkg_json_found), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): # Resource that exists out, e = self.ckan.package_exists(name='urbanpark') assert out is not False, "It should return a pkg" err_msg = "The pkg id should be the same" assert out['id'] == self.pkg_json_found['result']['id'], err_msg @patch('pybossa.ckan.requests.get') def test_02_get_resource_id(self, Mock): """Test CKAN get_resource_id works""" html_request = FakeRequest(json.dumps(self.pkg_json_found), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): # Resource that exists # Get the package out, e = self.ckan.package_exists(name='urbanpark') # Get the resource id for Task out = self.ckan.get_resource_id(name='task') err_msg = "It should return the task resource ID" assert out == self.task_resource_id, err_msg # Get the resource id for TaskRun out = self.ckan.get_resource_id(name='task_run') err_msg = "It should return the task_run resource ID" assert out == self.task_run_resource_id, err_msg # Get the resource id for a non existant resource err_msg = "It should return false" out = self.ckan.get_resource_id(name='non-existant') assert out is False, err_msg @patch('pybossa.ckan.requests.post') def test_03_package_create(self, Mock): """Test CKAN package_create works""" # It should return self.pkg_json_found with an empty Resources list html_request = FakeRequest(json.dumps(self.pkg_json_found), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): # Resource that exists project = Project(short_name='urbanpark', name='Urban Parks') user = User(fullname='Daniel Lombrana Gonzalez') out = self.ckan.package_create(project=project, user=user, url="http://something.com") err_msg = "The package ID should be the same" assert out['id'] == self.package_id, err_msg # Check the exception Mock.return_value = self.server_error try: self.ckan.package_create(project=project, user=user, url="http://something.com") except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! package_create failed" == type, type @patch('pybossa.ckan.requests.post') def test_05_resource_create(self, Mock): """Test CKAN resource_create works""" pkg_request = FakeRequest(json.dumps(self.pkg_json_found), 200, {'content-type': 'application/json'}) rsrc_request = FakeRequest( json.dumps(self.pkg_json_found['result']['resources'][0]), 200, {'content-type': 'text/html'}) Mock.return_value = pkg_request with self.flask_app.test_request_context('/'): # Resource that exists project = Project(short_name='urbanpark', name='Urban Parks') user = User(fullname='Daniel Lombrana Gonzalez') self.ckan.package_create(project=project, user=user, url="http://something.com") Mock.return_value = rsrc_request out = self.ckan.resource_create(name='task') err_msg = "It should create the task resource" assert out["id"] == self.task_resource_id, err_msg Mock.return_value = self.server_error try: self.ckan.resource_create(name='something-goes-wrong') except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! resource_create failed" == type, type @patch('pybossa.ckan.requests.post') def test_05_datastore_create_without_resource_id(self, Mock): """Test CKAN datastore_create without resource_id works""" html_request = FakeRequest(json.dumps(self.task_datastore), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): out = self.ckan.datastore_create(name='task', resource_id=None) err_msg = "It should ref the task resource ID" assert out['resource_id'] == self.task_resource_id, err_msg # Check the error Mock.return_value = self.server_error try: self.ckan.datastore_create(name='task', resource_id=self.task_resource_id) except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, err_msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! datastore_create failed" == type, type @patch('pybossa.ckan.requests.post') def test_05_datastore_create(self, Mock): """Test CKAN datastore_create works""" html_request = FakeRequest(json.dumps(self.task_datastore), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): out = self.ckan.datastore_create(name='task', resource_id=self.task_resource_id) err_msg = "It should ref the task resource ID" assert out['resource_id'] == self.task_resource_id, err_msg # Check the error Mock.return_value = self.server_error try: self.ckan.datastore_create(name='task', resource_id=self.task_resource_id) except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, err_msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! datastore_create failed" == type, type @patch('pybossa.ckan.requests.post') def test_06_datastore_upsert_without_resource_id(self, Mock): """Test CKAN datastore_upsert without resourece_id works""" html_request = FakeRequest(json.dumps(self.task_upsert), 200, {'content-type': 'application/json'}) record = dict(info=dict(foo="bar")) Mock.return_value = html_request with self.flask_app.test_request_context('/'): out = self.ckan.datastore_upsert(name='task', records=json.dumps([record]), resource_id=None) err_msg = "It should return True" assert out is True, err_msg # Check the error Mock.return_value = self.server_error try: self.ckan.datastore_upsert(name='task', records=json.dumps([record]), resource_id=self.task_resource_id) except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! datastore_upsert failed" == type, type @patch('pybossa.ckan.requests.post') def test_06_datastore_upsert(self, Mock): """Test CKAN datastore_upsert works""" html_request = FakeRequest(json.dumps(self.task_upsert), 200, {'content-type': 'application/json'}) record = dict(info=dict(foo="bar")) Mock.return_value = html_request with self.flask_app.test_request_context('/'): out = self.ckan.datastore_upsert(name='task', records=json.dumps([record]), resource_id=self.task_resource_id) err_msg = "It should return True" assert out is True, err_msg # Check the error Mock.return_value = self.server_error try: self.ckan.datastore_upsert(name='task', records=json.dumps([record]), resource_id=self.task_resource_id) except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! datastore_upsert failed" == type, type @patch('pybossa.ckan.requests.post') def test_07_datastore_delete(self, Mock): """Test CKAN datastore_delete works""" html_request = FakeRequest(json.dumps({}), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): out = self.ckan.datastore_delete(name='task', resource_id=self.task_resource_id) err_msg = "It should return True" assert out is True, err_msg # Check the error Mock.return_value = self.server_error try: self.ckan.datastore_delete(name='task', resource_id=self.task_resource_id) except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! datastore_delete failed" == type, type @patch('pybossa.ckan.requests.post') def test_08_package_update(self, Mock): """Test CKAN package_update works""" html_request = FakeRequest(json.dumps(self.pkg_json_found), 200, {'content-type': 'application/json'}) Mock.return_value = html_request with self.flask_app.test_request_context('/'): # Resource that exists project = Project(short_name='urbanpark', name='Urban Parks') user = User(fullname='Daniel Lombrana Gonzalez') out = self.ckan.package_update( project=project, user=user, url="http://something.com", resources=self.pkg_json_found['result']['resources']) err_msg = "The package ID should be the same" assert out['id'] == self.package_id, err_msg # Check the exception Mock.return_value = self.server_error try: self.ckan.package_update( project=project, user=user, url="http://something.com", resources=self.pkg_json_found['result']['resources']) except Exception as out: type, msg, status_code = out.args assert "Server Error" in msg, msg assert 500 == status_code, status_code assert "CKAN: the remote site failed! package_update failed" == type, type
def respond_ckan(ty): # First check if there is a package (dataset) in CKAN tables = {"task": model.Task, "task_run": model.TaskRun} msg_1 = gettext("Data exported to ") msg = msg_1 + "%s ..." % current_app.config['CKAN_URL'] ckan = Ckan(url=current_app.config['CKAN_URL'], api_key=current_user.ckan_api) app_url = url_for('.details', short_name=app.short_name, _external=True) try: package, e = ckan.package_exists(name=app.short_name) if e: raise e if package: # Update the package package = ckan.package_update(app=app, user=app.owner, url=app_url, resources=package['resources']) ckan.package = package resource_found = False print len(package['resources']) for r in package['resources']: if r['name'] == ty: ckan.datastore_delete(name=ty, resource_id=r['id']) ckan.datastore_create(name=ty, resource_id=r['id']) ckan.datastore_upsert(name=ty, records=gen_json(tables[ty]), resource_id=r['id']) resource_found = True break if not resource_found: create_ckan_datastore(ckan, ty, package['id']) else: package = ckan.package_create(app=app, user=app.owner, url=app_url) create_ckan_datastore(ckan, ty, package['id']) #new_resource = ckan.resource_create(name=ty, # package_id=package['id']) #ckan.datastore_create(name=ty, # resource_id=new_resource['result']['id']) #ckan.datastore_upsert(name=ty, # records=gen_json(tables[ty]), # resource_id=new_resource['result']['id']) flash(msg, 'success') return respond() except requests.exceptions.ConnectionError: msg = "CKAN server seems to be down, try again layer or contact the CKAN admins" current_app.logger.error(msg) flash(msg, 'danger') except Exception as inst: if len(inst.args) == 3: t, msg, status_code = inst.args msg = ("Error: %s with status code: %s" % (t, status_code)) else: msg = ("Error: %s" % inst.args[0]) current_app.logger.error(msg) flash(msg, 'danger') finally: return respond()
def respond_ckan(ty): # First check if there is a package (dataset) in CKAN tables = {"task": model.Task, "task_run": model.TaskRun} msg_1 = lazy_gettext("Data exported to ") msg = msg_1 + "%s ..." % current_app.config['CKAN_URL'] ckan = Ckan(url=current_app.config['CKAN_URL'], api_key=current_user.ckan_api) app_url = url_for('.details', short_name=app.short_name, _external=True) try: package = ckan.package_exists(name=app.short_name) if package: # Update the package ckan.package_update(app=app, user=app.owner, url=app_url) if len(package['resources']) == 0: resources = create_ckan_datastores(ckan) ckan.datastore_upsert(name=ty, records=gen_json(tables[ty]), resource_id=resources[ty]['id']) flash(msg, 'success') return render_template('/applications/export.html', title=title, app=app) else: ckan.datastore_delete(name=ty) ckan.datastore_create(name=ty) ckan.datastore_upsert(name=ty, records=gen_json(tables[ty])) flash(msg, 'success') return render_template('/applications/export.html', title=title, app=app) else: ckan.package_create(app=app, user=app.owner, url=app_url, tags=current_app.config['BRAND']) resources = create_ckan_datastores(ckan) ckan.datastore_upsert(name=ty, records=gen_json(tables[ty]), resource_id=resources[ty]['id']) flash(msg, 'success') return render_template('/applications/export.html', title=title, app=app) except Exception as inst: print inst if len(inst.args) == 3: type, msg, status_code = inst.args msg = ("Error: %s with status code: %s" % (type, status_code)) else: msg = ("Error: %s" % inst.args[0]) flash(msg, 'danger') return render_template('/applications/export.html', title=title, app=app)
def respond_ckan(ty): # First check if there is a package (dataset) in CKAN tables = {"task": model.Task, "task_run": model.TaskRun} msg_1 = lazy_gettext("Data exported to ") msg = msg_1 + "%s ..." % current_app.config['CKAN_URL'] ckan = Ckan(url=current_app.config['CKAN_URL'], api_key=current_user.ckan_api) app_url = url_for('.details', short_name=app.short_name, _external=True) try: package, e = ckan.package_exists(name=app.short_name) if e: raise e if package: # Update the package ckan.package_update(app=app, user=app.owner, url=app_url) if len(package['resources']) == 0: resources = create_ckan_datastores(ckan) ckan.datastore_upsert(name=ty, records=gen_json(tables[ty]), resource_id=resources[ty]['id']) flash(msg, 'success') return render_template('/applications/export.html', title=title, loading_text=loading_text, app=app) else: ckan.datastore_delete(name=ty) ckan.datastore_create(name=ty) ckan.datastore_upsert(name=ty, records=gen_json(tables[ty])) flash(msg, 'success') return render_template('/applications/export.html', title=title, loading_text=loading_text, app=app) else: ckan.package_create(app=app, user=app.owner, url=app_url, tags=current_app.config['BRAND']) resources = create_ckan_datastores(ckan) ckan.datastore_upsert(name=ty, records=gen_json(tables[ty]), resource_id=resources[ty]['id']) flash(msg, 'success') return render_template('/applications/export.html', title=title, loading_text=loading_text, app=app) except requests.exceptions.ConnectionError: msg = "CKAN server seems to be down, try again layer or contact the CKAN admins" current_app.logger.error(msg) flash(msg, 'danger') except Exception as inst: if len(inst.args) == 3: type, msg, status_code = inst.args msg = ("Error: %s with status code: %s" % (type, status_code)) current_app.logger.error(msg) else: msg = ("Error: %s" % inst.args[0]) current_app.logger.error(msg) flash(msg, 'danger') finally: return render_template('/applications/export.html', title=title, loading_text=loading_text, app=app)