Exemplo n.º 1
0
def details(short_name):
    app = app_by_shortname(short_name)

    try:
        require.app.read(app)
        template = '/applications/app.html'
    except HTTPException:
        if app.hidden:
            raise abort(403)
        else:
            raise

    title = app_title(app, None)

    template_args = {"app": app, "title": title}
    try:
        if current_app.config.get('CKAN_URL'):
            template_args['ckan_name'] = current_app.config.get('CKAN_NAME')
            ckan = Ckan(url=current_app.config['CKAN_URL'])
            pkg, e = ckan.package_exists(name=short_name)
            if e:
                raise e
            if pkg:
                template_args['ckan_pkg_url'] = (
                    "%s/dataset/%s" % (current_app.config['CKAN_URL'], short_name))
                template_args['ckan_pkg'] = pkg
    except requests.exceptions.ConnectionError:
        current_app.logger.error("CKAN server down or there is a typo in the URL")
    except Exception as e:
        current_app.logger.error(e)

    return render_template(template, **template_args)
Exemplo n.º 2
0
    def respond_ckan(ty):
        # First check if there is a package (dataset) in CKAN
        tables = {"task": model.Task, "task_run": model.TaskRun}
        msg_1 = gettext("Data exported to ")
        msg = msg_1 + "%s ..." % current_app.config['CKAN_URL']
        ckan = Ckan(url=current_app.config['CKAN_URL'],
                    api_key=current_user.ckan_api)
        app_url = url_for('.details', short_name=app.short_name, _external=True)

        try:
            package, e = ckan.package_exists(name=app.short_name)
            if e:
                raise e
            if package:
                # Update the package
                owner = User.query.get(app.owner_id)
                package = ckan.package_update(app=app, user=owner, url=app_url,
                                              resources=package['resources'])

                ckan.package = package
                resource_found = False
                # print len(package['resources'])
                for r in package['resources']:
                    if r['name'] == ty:
                        ckan.datastore_delete(name=ty, resource_id=r['id'])
                        ckan.datastore_create(name=ty, resource_id=r['id'])
                        ckan.datastore_upsert(name=ty,
                                              records=gen_json(tables[ty]),
                                              resource_id=r['id'])
                        resource_found = True
                        break
                if not resource_found:
                    create_ckan_datastore(ckan, ty, package['id'])
            else:
                owner = User.query.get(app.owner_id)
                package = ckan.package_create(app=app, user=owner, url=app_url)
                create_ckan_datastore(ckan, ty, package['id'])
                #new_resource = ckan.resource_create(name=ty,
                #                                    package_id=package['id'])
                #ckan.datastore_create(name=ty,
                #                      resource_id=new_resource['result']['id'])
                #ckan.datastore_upsert(name=ty,
                #                     records=gen_json(tables[ty]),
                #                     resource_id=new_resource['result']['id'])
            flash(msg, 'success')
            return respond()
        except requests.exceptions.ConnectionError:
                msg = "CKAN server seems to be down, try again layer or contact the CKAN admins"
                current_app.logger.error(msg)
                flash(msg, 'danger')
        except Exception as inst:
            if len(inst.args) == 3:
                t, msg, status_code = inst.args
                msg = ("Error: %s with status code: %s" % (t, status_code))
            else: # pragma: no cover
                msg = ("Error: %s" % inst.args[0])
            current_app.logger.error(msg)
            flash(msg, 'danger')
        finally:
            return respond()
Exemplo n.º 3
0
    def respond_ckan(ty):
        # First check if there is a package (dataset) in CKAN
        tables = {"task": model.Task, "task_run": model.TaskRun}
        msg_1 = lazy_gettext("Data exported to ")
        msg = msg_1 + "%s ..." % current_app.config['CKAN_URL']
        ckan = Ckan(url=current_app.config['CKAN_URL'],
                    api_key=current_user.ckan_api)
        app_url = url_for('.details',
                          short_name=app.short_name,
                          _external=True)

        try:
            package = ckan.package_exists(name=app.short_name)
            if package:
                # Update the package
                ckan.package_update(app=app, user=app.owner, url=app_url)
                if len(package['resources']) == 0:
                    resources = create_ckan_datastores(ckan)
                    ckan.datastore_upsert(name=ty,
                                          records=gen_json(tables[ty]),
                                          resource_id=resources[ty]['id'])
                    flash(msg, 'success')
                    return render_template('/applications/export.html',
                                           title=title,
                                           app=app)
                else:
                    ckan.datastore_delete(name=ty)
                    ckan.datastore_create(name=ty)
                    ckan.datastore_upsert(name=ty,
                                          records=gen_json(tables[ty]))
                    flash(msg, 'success')
                    return render_template('/applications/export.html',
                                           title=title,
                                           app=app)
            else:
                ckan.package_create(app=app,
                                    user=app.owner,
                                    url=app_url,
                                    tags=current_app.config['BRAND'])
                resources = create_ckan_datastores(ckan)
                ckan.datastore_upsert(name=ty,
                                      records=gen_json(tables[ty]),
                                      resource_id=resources[ty]['id'])

                flash(msg, 'success')
                return render_template('/applications/export.html',
                                       title=title,
                                       app=app)
        except Exception as inst:
            print inst
            if len(inst.args) == 3:
                type, msg, status_code = inst.args
                msg = ("Error: %s with status code: %s" % (type, status_code))
            else:
                msg = ("Error: %s" % inst.args[0])
            flash(msg, 'danger')
            return render_template('/applications/export.html',
                                   title=title,
                                   app=app)
Exemplo n.º 4
0
    def respond_ckan(ty):
        # First check if there is a package (dataset) in CKAN
        tables = {"task": model.Task, "task_run": model.TaskRun}
        msg_1 = lazy_gettext("Data exported to ")
        msg = msg_1 + "%s ..." % current_app.config['CKAN_URL']
        ckan = Ckan(url=current_app.config['CKAN_URL'],
                    api_key=current_user.ckan_api)
        app_url = url_for('.details',
                          short_name=app.short_name,
                          _external=True)

        try:
            package, e = ckan.package_exists(name=app.short_name)
            if e:
                raise e
            if package:
                # Update the package
                ckan.package_update(app=app, user=app.owner, url=app_url)
                if len(package['resources']) == 0:
                    resources = create_ckan_datastores(ckan)
                    ckan.datastore_upsert(name=ty,
                                          records=gen_json(tables[ty]),
                                          resource_id=resources[ty]['id'])
                else:
                    ckan.datastore_delete(name=ty)
                    ckan.datastore_create(name=ty)
                    ckan.datastore_upsert(name=ty,
                                          records=gen_json(tables[ty]))
            else:
                ckan.package_create(app=app,
                                    user=app.owner,
                                    url=app_url,
                                    tags=current_app.config['BRAND'])
                resources = create_ckan_datastores(ckan)
                ckan.datastore_upsert(name=ty,
                                      records=gen_json(tables[ty]),
                                      resource_id=resources[ty]['id'])

            flash(msg, 'success')
            return respond()
        except requests.exceptions.ConnectionError:
            msg = "CKAN server seems to be down, try again layer or contact the CKAN admins"
            current_app.logger.error(msg)
            flash(msg, 'danger')
        except Exception as inst:
            if len(inst.args) == 3:
                type, msg, status_code = inst.args
                msg = ("Error: %s with status code: %s" % (type, status_code))
            else:
                msg = ("Error: %s" % inst.args[0])
            current_app.logger.error(msg)
            flash(msg, 'danger')
        finally:
            return respond()
Exemplo n.º 5
0
def details(short_name):
    app = app_by_shortname(short_name)

    try:
        require.app.read(app)
        require.app.update(app)
        template = '/applications/actions.html'
    except HTTPException:
        if app.hidden:
            app = None
        template = '/applications/app.html'

    title = app_title(app, None)
    template_args = {"app": app, "title": title}
    if current_app.config.get('CKAN_URL'):
        template_args['ckan_name'] = current_app.config.get('CKAN_NAME')
        ckan = Ckan(url=current_app.config['CKAN_URL'])
        pkg = ckan.package_exists(name=short_name)
        if pkg:
            template_args['ckan_pkg_url'] = (
                "%s/dataset/%s" % (current_app.config['CKAN_URL'], short_name))
            template_args['ckan_pkg'] = pkg
    return render_template(template, **template_args)
Exemplo n.º 6
0
class TestCkanModule(Test, object):

    ckan = Ckan(url="http://datahub.io", api_key="fake-api-key")
    task_resource_id = "0dde48c7-a0e9-445f-bc84-6365ec057450"
    task_run_resource_id = "a49448dc-228a-4d54-a697-ba02c50e0143"
    package_id = "4bcf844c-3ad0-4203-8418-32e7d7c4ce96"
    pkg_json_not_found = {
        "help": "Return ...",
        "success": False,
        "error": {
            "message": "Not found",
            "__type": "Not Found Error"
        }
    }

    pkg_json_found = {
        "help": "Return the metadata of a dataset ...",
        "success": True,
        "result": {
            "license_title":
            "",
            "maintainer":
            "",
            "relationships_as_object": [],
            "maintainer_email":
            "",
            "revision_timestamp":
            "2013-04-11T11:45:52.689160",
            "id":
            package_id,
            "metadata_created":
            "2013-04-11T11:39:56.003541",
            "metadata_modified":
            "2013-04-12T10:50:45.132825",
            "author":
            "Daniel Lombrana Gonzalez",
            "author_email":
            "",
            "state":
            "deleted",
            "version":
            "",
            "license_id":
            "",
            "type":
            None,
            "resources": [{
                "resource_group_id": "f45c29ce-97f3-4b1f-b060-3306ffedb64b",
                "cache_last_updated": None,
                "revision_timestamp": "2013-04-12T10:50:41.635556",
                "webstore_last_updated": None,
                "id": task_resource_id,
                "size": None,
                "state": "active",
                "last_modified": None,
                "hash": "",
                "description": "tasks",
                "format": "",
                "tracking_summary": {
                    "total": 0,
                    "recent": 0
                },
                "mimetype_inner": None,
                "mimetype": None,
                "cache_url": None,
                "name": "task",
                "created": "2013-04-12T05:50:41.776512",
                "url": "http://*****:*****@patch('pybossa.ckan.requests.get')
    def test_00_package_exists_returns_false(self, Mock):
        """Test CKAN get_resource_id works"""
        html_request = FakeRequest(json.dumps(self.pkg_json_not_found), 200,
                                   {'content-type': 'application/json'})
        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            # Resource that exists
            out, e = self.ckan.package_exists(name='not-found')
            assert out is False, "It should return False as pkg does not exist"
            # Handle error in CKAN server
            Mock.return_value = self.server_error
            try:
                pkg, e = self.ckan.package_exists(name="something-goes-wrong")
                if e:
                    raise e
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, msg
                assert status_code == 500, "status_code should be 500"
                assert type == "CKAN: the remote site failed! package_show failed"
            # Now with a broken JSON item
            Mock.return_value = FakeRequest("simpletext", 200,
                                            {'content-type': 'text/html'})
            out, e = self.ckan.package_exists(name='not-found')
            assert out is False, "It should return False as pkg does not exist"
            # Handle error in CKAN server
            try:
                pkg, e = self.ckan.package_exists(name="something-goes-wrong")
                if e:
                    raise e
            except Exception as out:
                type, msg, status_code = out.args
                assert status_code == 200, "status_code should be 200"
                assert type == "CKAN: JSON not valid"

    @patch('pybossa.ckan.requests.get')
    def test_01_package_exists_returns_pkg(self, Mock):
        """Test CKAN get_resource_id works"""
        html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
                                   {'content-type': 'application/json'})
        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            # Resource that exists
            out, e = self.ckan.package_exists(name='urbanpark')
            assert out is not False, "It should return a pkg"
            err_msg = "The pkg id should be the same"
            assert out['id'] == self.pkg_json_found['result']['id'], err_msg

    @patch('pybossa.ckan.requests.get')
    def test_02_get_resource_id(self, Mock):
        """Test CKAN get_resource_id works"""
        html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
                                   {'content-type': 'application/json'})
        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            # Resource that exists
            # Get the package
            out, e = self.ckan.package_exists(name='urbanpark')
            # Get the resource id for Task
            out = self.ckan.get_resource_id(name='task')
            err_msg = "It should return the task resource ID"
            assert out == self.task_resource_id, err_msg
            # Get the resource id for TaskRun
            out = self.ckan.get_resource_id(name='task_run')
            err_msg = "It should return the task_run resource ID"
            assert out == self.task_run_resource_id, err_msg
            # Get the resource id for a non existant resource
            err_msg = "It should return false"
            out = self.ckan.get_resource_id(name='non-existant')
            assert out is False, err_msg

    @patch('pybossa.ckan.requests.post')
    def test_03_package_create(self, Mock):
        """Test CKAN package_create works"""
        # It should return self.pkg_json_found with an empty Resources list
        html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
                                   {'content-type': 'application/json'})
        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            # Resource that exists
            project = Project(short_name='urbanpark', name='Urban Parks')
            user = User(fullname='Daniel Lombrana Gonzalez')
            out = self.ckan.package_create(project=project,
                                           user=user,
                                           url="http://something.com")
            err_msg = "The package ID should be the same"
            assert out['id'] == self.package_id, err_msg

            # Check the exception
            Mock.return_value = self.server_error
            try:
                self.ckan.package_create(project=project,
                                         user=user,
                                         url="http://something.com")
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! package_create failed" == type, type

    @patch('pybossa.ckan.requests.post')
    def test_05_resource_create(self, Mock):
        """Test CKAN resource_create works"""
        pkg_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
                                  {'content-type': 'application/json'})

        rsrc_request = FakeRequest(
            json.dumps(self.pkg_json_found['result']['resources'][0]), 200,
            {'content-type': 'text/html'})
        Mock.return_value = pkg_request
        with self.flask_app.test_request_context('/'):
            # Resource that exists
            project = Project(short_name='urbanpark', name='Urban Parks')
            user = User(fullname='Daniel Lombrana Gonzalez')
            self.ckan.package_create(project=project,
                                     user=user,
                                     url="http://something.com")
            Mock.return_value = rsrc_request
            out = self.ckan.resource_create(name='task')
            err_msg = "It should create the task resource"
            assert out["id"] == self.task_resource_id, err_msg
            Mock.return_value = self.server_error
            try:
                self.ckan.resource_create(name='something-goes-wrong')
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! resource_create failed" == type, type

    @patch('pybossa.ckan.requests.post')
    def test_05_datastore_create_without_resource_id(self, Mock):
        """Test CKAN datastore_create without resource_id works"""
        html_request = FakeRequest(json.dumps(self.task_datastore), 200,
                                   {'content-type': 'application/json'})

        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            out = self.ckan.datastore_create(name='task', resource_id=None)
            err_msg = "It should ref the task resource ID"
            assert out['resource_id'] == self.task_resource_id, err_msg
            # Check the error
            Mock.return_value = self.server_error
            try:
                self.ckan.datastore_create(name='task',
                                           resource_id=self.task_resource_id)
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, err_msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! datastore_create failed" == type, type

    @patch('pybossa.ckan.requests.post')
    def test_05_datastore_create(self, Mock):
        """Test CKAN datastore_create works"""
        html_request = FakeRequest(json.dumps(self.task_datastore), 200,
                                   {'content-type': 'application/json'})

        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            out = self.ckan.datastore_create(name='task',
                                             resource_id=self.task_resource_id)
            err_msg = "It should ref the task resource ID"
            assert out['resource_id'] == self.task_resource_id, err_msg
            # Check the error
            Mock.return_value = self.server_error
            try:
                self.ckan.datastore_create(name='task',
                                           resource_id=self.task_resource_id)
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, err_msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! datastore_create failed" == type, type

    @patch('pybossa.ckan.requests.post')
    def test_06_datastore_upsert_without_resource_id(self, Mock):
        """Test CKAN datastore_upsert without resourece_id works"""
        html_request = FakeRequest(json.dumps(self.task_upsert), 200,
                                   {'content-type': 'application/json'})

        record = dict(info=dict(foo="bar"))
        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            out = self.ckan.datastore_upsert(name='task',
                                             records=json.dumps([record]),
                                             resource_id=None)
            err_msg = "It should return True"
            assert out is True, err_msg
            # Check the error
            Mock.return_value = self.server_error
            try:
                self.ckan.datastore_upsert(name='task',
                                           records=json.dumps([record]),
                                           resource_id=self.task_resource_id)
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! datastore_upsert failed" == type, type

    @patch('pybossa.ckan.requests.post')
    def test_06_datastore_upsert(self, Mock):
        """Test CKAN datastore_upsert works"""
        html_request = FakeRequest(json.dumps(self.task_upsert), 200,
                                   {'content-type': 'application/json'})

        record = dict(info=dict(foo="bar"))
        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            out = self.ckan.datastore_upsert(name='task',
                                             records=json.dumps([record]),
                                             resource_id=self.task_resource_id)
            err_msg = "It should return True"
            assert out is True, err_msg
            # Check the error
            Mock.return_value = self.server_error
            try:
                self.ckan.datastore_upsert(name='task',
                                           records=json.dumps([record]),
                                           resource_id=self.task_resource_id)
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! datastore_upsert failed" == type, type

    @patch('pybossa.ckan.requests.post')
    def test_07_datastore_delete(self, Mock):
        """Test CKAN datastore_delete works"""
        html_request = FakeRequest(json.dumps({}), 200,
                                   {'content-type': 'application/json'})

        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            out = self.ckan.datastore_delete(name='task',
                                             resource_id=self.task_resource_id)
            err_msg = "It should return True"
            assert out is True, err_msg
            # Check the error
            Mock.return_value = self.server_error
            try:
                self.ckan.datastore_delete(name='task',
                                           resource_id=self.task_resource_id)
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! datastore_delete failed" == type, type

    @patch('pybossa.ckan.requests.post')
    def test_08_package_update(self, Mock):
        """Test CKAN package_update works"""
        html_request = FakeRequest(json.dumps(self.pkg_json_found), 200,
                                   {'content-type': 'application/json'})
        Mock.return_value = html_request
        with self.flask_app.test_request_context('/'):
            # Resource that exists
            project = Project(short_name='urbanpark', name='Urban Parks')
            user = User(fullname='Daniel Lombrana Gonzalez')
            out = self.ckan.package_update(
                project=project,
                user=user,
                url="http://something.com",
                resources=self.pkg_json_found['result']['resources'])
            err_msg = "The package ID should be the same"
            assert out['id'] == self.package_id, err_msg

            # Check the exception
            Mock.return_value = self.server_error
            try:
                self.ckan.package_update(
                    project=project,
                    user=user,
                    url="http://something.com",
                    resources=self.pkg_json_found['result']['resources'])
            except Exception as out:
                type, msg, status_code = out.args
                assert "Server Error" in msg, msg
                assert 500 == status_code, status_code
                assert "CKAN: the remote site failed! package_update failed" == type, type