Beispiel #1
0
def load_from_databank(sourcejson, dataproviderjson, dry_run=False, overwrite=True, meta_only=False, file_dir = None):

    print "Working on ", sourcejson['fields']['indicator']


    dataorg = DataOrg.by_name(dataproviderjson['fields']['title'])

    dataorgMeta = {
        'description': dataproviderjson['fields']['description'], 
        'label': dataproviderjson['fields']['title']
    }

    if not dataorg:
        dataorg = DataOrg(dataorgMeta)
        db.session.add(dataorg)

    #dataorg will update with id here
    db.session.commit()

    #get or create dataset
    dataset = Dataset.by_label(sourcejson['fields']['indicator'])

    description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(sourcejson['pk']) + "/"

    modelDataset = {'dataset': 
            {
                'label': sourcejson['fields']['indicator'],
                'name': sourcejson['fields']['indicator'],
                'description': description,
                'dataType': sourcejson['fields']['data_type'],
                'dataorg_id': dataorg.id
            }
        }

    if not dataset:
        #create one

        dataset = Dataset(modelDataset['dataset'])
        #dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
        #dataset.data = dataproviderjson['fields'].get('mapping',{})
        db.session.add(dataset)

    else:
        #dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
        #dataset.data = dataproviderjson['fields'].get('mapping',{})
        dataset.update(modelDataset['dataset'])


    db.session.commit()


    systemaccount = Account.by_id(1)

    if dataset.source:
        try:
            print "trying to delete source"
            print dataset.source
            dataset.source.delete()
        except Exception, e:
            print "could not delete source", e
Beispiel #2
0
    def test_browser_for_entity(self):
        h.skip_if_stubbed_solr()

        from openspending.model import Dataset, Entry

        dataset = Dataset(name='testdataset')
        Dataset.c.save(dataset, manipulate=True)
        dataset_ref_dict = dataset.to_ref_dict()

        entity = self._make_one(name="Test Entity", label="Test Entity Label")
        entity_ref_dict = entity.to_ref_dict()

        entry = {'name': 'Test Entry',
                 'label': 'Test Entry Label',
                 'from': entity_ref_dict,
                 'to': entity_ref_dict,
                 'amount': 10.0,
                 'dataset': dataset_ref_dict}
        Entry.c.save(entry)

        h.clean_and_reindex_solr()

        entity_url = url(controller='entity', id=str(entity['_id']),
                         slug='test-entity-label', action='view')
        response = self.app.get(entity_url)

        h.assert_equal(response._status, '200 OK')
        h.assert_true('<b>1 entries</b> found.<br />' in response)
        h.assert_true('entries.json">' in response)
        h.assert_true('entries.csv">' in response)
Beispiel #3
0
 def create(self):
     require.dataset.create()
     try:
         dataset = dict(request.params)
         dataset['territories'] = request.params.getall('territories')
         dataset['languages'] = request.params.getall('languages')
         model = {'dataset': dataset}
         schema = dataset_schema(ValidationState(model))
         data = schema.deserialize(dataset)
         if Dataset.by_name(data['name']) is not None:
             raise Invalid(
                 SchemaNode(String(), name='dataset.name'),
                 _("A dataset with this identifer already exists!"))
         dataset = Dataset({'dataset': data})
         dataset.private = True
         dataset.managers.append(c.account)
         db.session.add(dataset)
         db.session.commit()
         redirect(
             h.url_for(controller='editor',
                       action='index',
                       dataset=dataset.name))
     except Invalid, i:
         errors = i.asdict()
         return self.new(errors)
Beispiel #4
0
    def create(self):
        """
        Adds a new dataset dynamically through a POST request
        """

        # User must be authenticated so we should have a user object in
        # c.account, if not abort with error message
        if not c.account:
            abort(status_code=400, detail='user not authenticated')

        # Check if the params are there ('metadata', 'csv_file')
        if len(request.params) != 2:
            abort(status_code=400, detail='incorrect number of params')

        metadata = request.params['metadata'] \
            if 'metadata' in request.params \
            else abort(status_code=400, detail='metadata is missing')

        csv_file = request.params['csv_file'] \
            if 'csv_file' in request.params \
            else abort(status_code=400, detail='csv_file is missing')

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            abort(status_code=400, detail='JSON model could not be parsed')
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            abort(status_code=400, detail='Model is not well formed')
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = True  # Default value
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account, url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))
Beispiel #5
0
    def create(self):
        """
        Adds a new dataset dynamically through a POST request
        """

        # User must be authenticated so we should have a user object in
        # c.account, if not abort with error message
        if not c.account:
            abort(status_code=400, detail='user not authenticated')

        # Check if the params are there ('metadata', 'csv_file')
        if len(request.params) != 2:
            abort(status_code=400, detail='incorrect number of params')

        metadata = request.params['metadata'] \
            if 'metadata' in request.params \
            else abort(status_code=400, detail='metadata is missing')

        csv_file = request.params['csv_file'] \
            if 'csv_file' in request.params \
            else abort(status_code=400, detail='csv_file is missing')

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            abort(status_code=400, detail='JSON model could not be parsed')
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            abort(status_code=400, detail='Model is not well formed')
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = True  # Default value
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account, url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))
Beispiel #6
0
 def test_views_update(self):
     cra = Dataset.by_name('cra')
     views = cra.data['views']
     views[0]['label'] = 'Banana'
     response = self.app.post(url(controller='editor',
                                  action='views_update', dataset='cra'),
                              params={'views': json.dumps(views)},
                              extra_environ={'REMOTE_USER': '******'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert 'Banana' in repr(cra.data['views'])
Beispiel #7
0
 def test_views_update(self):
     cra = Dataset.by_name('cra')
     views = cra.data['views']
     views[0]['label'] = 'Banana'
     response = self.app.post(url(controller='editor', 
         action='views_update', dataset='cra'),
         params={'views': json.dumps(views)},
         extra_environ={'REMOTE_USER': '******'},
         expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert 'Banana' in repr(cra.data['views'])
Beispiel #8
0
 def test_retract(self):
     cra = Dataset.by_name('cra')
     assert cra.private is False, cra.private
     response = self.app.post(url(controller='editor', 
         action='retract', dataset='cra'),
         extra_environ={'REMOTE_USER': '******'})
     cra = Dataset.by_name('cra')
     assert cra.private is True, cra.private
     response = self.app.post(url(controller='editor', 
         action='retract', dataset='cra'),
         extra_environ={'REMOTE_USER': '******'},
         expect_errors=True)
     assert '400' in response.status, response.status
Beispiel #9
0
 def test_retract(self):
     cra = Dataset.by_name('cra')
     assert cra.private is False, cra.private
     response = self.app.post(url(controller='editor',
                                  action='retract', dataset='cra'),
                              extra_environ={'REMOTE_USER': '******'})
     cra = Dataset.by_name('cra')
     assert cra.private is True, cra.private
     response = self.app.post(url(controller='editor',
                                  action='retract', dataset='cra'),
                              extra_environ={'REMOTE_USER': '******'},
                              expect_errors=True)
     assert '400' in response.status, response.status
class TestCompoundDimension(DatabaseTestCase):

    def setup(self):
        super(TestCompoundDimension, self).setup()
        self.engine = db.engine
        self.meta = db.metadata
        self.meta.bind = self.engine
        self.ds = Dataset(SIMPLE_MODEL)
        self.entity = self.ds['to']
        self.classifier = self.ds['function']

    def test_is_compound(self):
        h.assert_true(self.entity.is_compound)

    def test_basic_properties(self):
        assert self.entity.name=='to', self.entity.name
        assert self.classifier.name=='function', self.classifier.name

    def test_generated_tables(self):
        #assert not hasattr(self.entity, 'table'), self.entity
        #self.ds.generate()
        assert hasattr(self.entity, 'table'), self.entity
        assert self.entity.table.name=='test__' + self.entity.taxonomy, self.entity.table.name
        assert hasattr(self.entity, 'alias')
        assert self.entity.alias.name==self.entity.name, self.entity.alias.name
        cols = self.entity.table.c
        assert 'id' in cols
        assert_raises(KeyError, cols.__getitem__, 'field')

    def test_attributes_exist_on_object(self):
        assert len(self.entity.attributes)==2, self.entity.attributes
        assert_raises(KeyError, self.entity.__getitem__, 'field')
        assert self.entity['name'].name=='name'
        assert self.entity['name'].datatype=='id'

    def test_attributes_exist_on_table(self):
        assert hasattr(self.entity, 'table'), self.entity
        assert 'name' in self.entity.table.c, self.entity.table.c
        assert 'label' in self.entity.table.c, self.entity.table.c

    def test_members(self):
        self.ds.generate()
        self.entity.load(self.ds.bind, {'name': 'one', 'label': 'Label One'})
        self.entity.load(self.ds.bind, {'name': 'two', 'label': 'Label Two'})

        members = list(self.entity.members())
        h.assert_equal(len(members), 2)

        members = list(self.entity.members(self.entity.alias.c.name == 'one'))
        h.assert_equal(len(members), 1)
Beispiel #11
0
 def test_publish(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.post(url(controller='editor',
                                  action='publish', dataset='cra'),
                              extra_environ={'REMOTE_USER': '******'})
     cra = Dataset.by_name('cra')
     assert cra.private is False, cra.private
     response = self.app.post(url(controller='editor',
                                  action='publish', dataset='cra'),
                              extra_environ={'REMOTE_USER': '******'},
                              expect_errors=True)
     assert '400' in response.status, response.status
Beispiel #12
0
 def test_publish(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.post(url(controller='editor', 
         action='publish', dataset='cra'),
         extra_environ={'REMOTE_USER': '******'})
     cra = Dataset.by_name('cra')
     assert cra.private is False, cra.private
     response = self.app.post(url(controller='editor', 
         action='publish', dataset='cra'),
         extra_environ={'REMOTE_USER': '******'},
         expect_errors=True)
     assert '400' in response.status, response.status
Beispiel #13
0
def csvimport_fixture(name):
    model_fp = csvimport_fixture_file(name, 'model.json')
    mapping_fp = csvimport_fixture_file(name, 'mapping.json')
    model = json.load(model_fp)
    if mapping_fp:
        model['mapping'] = json.load(mapping_fp)
    dataset = Dataset(model)
    dataset.generate()
    db.session.add(dataset)
    data_path = csvimport_fixture_path(name, 'data.csv')
    user = make_account()
    source = Source(dataset, user, data_path)
    db.session.add(source)
    db.session.commit()
    return source
Beispiel #14
0
def csvimport_fixture(name):
    model_fp = csvimport_fixture_file(name, 'model.json')
    mapping_fp = csvimport_fixture_file(name, 'mapping.json')
    model = json.load(model_fp)
    if mapping_fp:
        model['mapping'] = json.load(mapping_fp)
    dataset = Dataset(model)
    dataset.generate()
    db.session.add(dataset)
    data_path = csvimport_fixture_path(name, 'data.csv')
    user = h.make_account()
    source = Source(dataset, user, data_path)
    db.session.add(source)
    db.session.commit()
    return source
Beispiel #15
0
def archive_one(dataset_name, archive_dir):
    """
    Find the dataset, create the archive directory and start archiving
    """

    # Find the dataset
    dataset = Dataset.by_name(dataset_name)
    # If no dataset found, exit with error message
    if dataset is None:
        exit_with_error("Dataset not found. Unable to archive it.")

    # If the archive_dir exists we have to ask the user if we should overwrite
    if os.path.exists(archive_dir):
        # If user doesn't want to write over it we exit
        if not get_confirmation("%s exists. Do you want to overwrite?"
                                % archive_dir):
            sys.exit(0)
        # If the archive dir is a file we don't do anything
        if os.path.isfile(archive_dir):
            exit_with_error("Cannot overwrite a file (need a directory).")
    # If the archive_dir doesn't exist we create it
    else:
        try:
            os.makedirs(archive_dir)
        except OSError:
            # If we couldn't create it, we exit with an error message
            exit_with_error("Couldn't create archive directory.")

    # Archive the model (dataset metadata)
    archive_model(dataset, archive_dir)
    # Archive the visualisations
    archive_visualisations(dataset, archive_dir)
    # Download all sources
    update(os.path.join(archive_dir, 'sources'), dataset)
Beispiel #16
0
    def test_drop(self):
        cra = Dataset.by_name('cra')
        assert len(cra)==36, len(cra)
        # double-check authz
        response = self.app.post(url(controller='editor', 
            action='drop', dataset='cra'),
            expect_errors=True)
        assert '403' in response.status
        cra = Dataset.by_name('cra')
        assert len(cra)==36, len(cra)

        response = self.app.post(url(controller='editor', 
            action='drop', dataset='cra'),
            extra_environ={'REMOTE_USER': '******'})
        cra = Dataset.by_name('cra')
        assert len(cra)==0, len(cra)
Beispiel #17
0
    def create_view(self, cls, add_filters, name, label, dimension,
                    breakdown=None, view_filters={}):
        '''\
        Create a view. The view will be computed when you call
        :meth:`finalize`.

        ``cls``
            A model class (inheriting from :class:`openspending.model.Base`)
        ``add_filters``
            A :term:`mongodb query spec` used as a query to select the
            instances of *cls* that will be used for the view.
        ``name``
            A name for the view. This name must be unique for all views
            for *cls* in an Open Spending site.
        ``label``
            A label that can be displayed to the user.
        ``dimensions``
            The dimensions that will be used to compute the view
        ``breakdown``
            ...
        ``view_filters``
            ...

        Returns: A :class:`openspending.lib.views.View` object.
        '''
        log.debug("pre-aggregating view %s on %r where %r",
                  name, cls, view_filters)
        view = View(self.dataset, name, label, dimension,
                    breakdown, cuts=view_filters)
        view.apply_to(cls, add_filters)
        view.compute()
        Dataset.c.update({'name': self.dataset.name},
                         {'$set': {'cubes': self.dataset.get('cubes', {})}})
        self.dataset = Dataset.by_id(self.dataset.name)
        return view
 def setup(self):
     self.engine = db.engine
     self.meta = db.metadata  #MetaData()
     self.meta.bind = self.engine
     self.ds = Dataset(SIMPLE_MODEL)
     self.entity = self.ds['to']
     self.classifier = self.ds['function']
Beispiel #19
0
 def setup(self):
     super(TestAttributeDimension, self).setup()
     self.engine = db.engine
     self.meta = db.metadata
     self.meta.bind = self.engine
     self.ds = Dataset(model_fixture('simple'))
     self.field = self.ds['field']
Beispiel #20
0
 def setup(self):
     super(TestAttributeDimension, self).setup()
     self.engine = db.engine
     self.meta = db.metadata
     self.meta.bind = self.engine
     self.ds = Dataset(SIMPLE_MODEL)
     self.field = self.ds['field']
    def test_new_wrong_user(self):
        # First we add a Dataset with user 'test_new'
        user = Account.by_name('test_new')
        assert user.api_key == 'd0610659-627b-4403-8b7f-6e2820ebc95d'

        u = url(controller='api/version2', action='create')
        params = {
            'metadata':
            'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
            'csv_file':
            'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
        }
        apikey_header = 'apikey {0}'.format(user.api_key)
        response = self.app.post(u, params, {'Authorization': apikey_header})

        assert "200" in response.status
        assert Dataset.by_name('openspending-example')

        # After that we try to update the Dataset with user 'test_new2'
        user = Account.by_name('test_new2')
        assert user.api_key == 'c011c340-8dad-419c-8138-1c6ded86ead5'

        u = url(controller='api/version2', action='create')
        params = {
            'metadata':
            'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
            'csv_file':
            'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
        }
        apikey_header = 'apikey {0}'.format(user.api_key)
        response = self.app.post(u,
                                 params, {'Authorization': apikey_header},
                                 expect_errors=True)
        assert '403' in response.status
    def test_new_wrong_user(self):
        # First we add a Dataset with user 'test_new'
        user = Account.by_name('test_new')
        assert user.api_key == 'd0610659-627b-4403-8b7f-6e2820ebc95d'

        u = url(controller='api/version2', action='create')
        params = {
            'metadata':
                'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
            'csv_file':
                'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
            }
        apikey_header = 'apikey {0}'.format(user.api_key)
        response = self.app.post(u, params, {'Authorization':apikey_header})
        #Dataset.by_name('openspending-example').private = False
        assert "200" in response.status
        assert Dataset.by_name('openspending-example')

        # After that we try to update the Dataset with user 'test_new2'
        user = Account.by_name('test_new2')
        assert user.api_key == 'c011c340-8dad-419c-8138-1c6ded86ead5'
        
        u = url(controller='api/version2', action='create')
        params = {
            'metadata':
                'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
            'csv_file':
                'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
            }
        apikey_header = 'apikey {0}'.format(user.api_key)
        response = self.app.post(u, params, {'Authorization':apikey_header},
                                 expect_errors=True)
        assert '403' in response.status    
Beispiel #23
0
    def permissions(self):
        """
        Check a user's permissions for a given dataset. This could also be
        done via request to the user, but since we're not really doing a
        RESTful service we do this via the api instead.
        """

        # Check the parameters. Since we only use one parameter we check it
        # here instead of creating a specific parameter parser
        if len(request.params) != 1 or 'dataset' not in request.params:
            return to_jsonp({'error': 'Parameter dataset missing'})

        # Get the dataset we want to check permissions for
        dataset = Dataset.by_name(request.params['dataset'])

        # Return permissions
        return to_jsonp({
            "create":
            can.dataset.create() and dataset is None,
            "read":
            False if dataset is None else can.dataset.read(dataset),
            "update":
            False if dataset is None else can.dataset.update(dataset),
            "delete":
            False if dataset is None else can.dataset.delete(dataset)
        })
Beispiel #24
0
    def index(self):
        # Get all of the datasets available to the account of the logged in
        # or an anonymous user (if c.account is None)
        c.datasets = Dataset.all_by_account(c.account)

        c.num_entries = dataset_entries(None)
        return templating.render('home/index.html')
Beispiel #25
0
def remove_dataset(dataset_name):
    log.warn("Dropping dataset '%s'", dataset_name)
    from openspending.model import Dataset, meta as db
    dataset = Dataset.by_name(dataset_name)
    dataset.drop()
    db.session.delete(dataset)
    db.session.commit()
Beispiel #26
0
def archive_one(dataset_name, archive_dir):
    """
    Find the dataset, create the archive directory and start archiving
    """

    # Find the dataset
    dataset = Dataset.by_name(dataset_name)
    # If no dataset found, exit with error message
    if dataset is None:
        exit_with_error("Dataset not found. Unable to archive it.")

    # If the archive_dir exists we have to ask the user if we should overwrite
    if os.path.exists(archive_dir):
        # If user doesn't want to write over it we exit
        if not get_confirmation(
                "%s exists. Do you want to overwrite?" % archive_dir):
            sys.exit(0)
        # If the archive dir is a file we don't do anything
        if os.path.isfile(archive_dir):
            exit_with_error("Cannot overwrite a file (need a directory).")
    # If the archive_dir doesn't exist we create it
    else:
        try:
            os.makedirs(archive_dir)
        except OSError:
            # If we couldn't create it, we exit with an error message
            exit_with_error("Couldn't create archive directory.")

    # Archive the model (dataset metadata)
    archive_model(dataset, archive_dir)
    # Archive the visualisations
    archive_visualisations(dataset, archive_dir)
    # Download all sources
    update(os.path.join(archive_dir, 'sources'), dataset)
Beispiel #27
0
 def test_index_hide_private(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.get(url(controller='dataset', action='index', format='json'))
     obj = json.loads(response.body)
     h.assert_equal(len(obj), 0)
Beispiel #28
0
    def permissions(self):
        """
        Check a user's permissions for a given dataset. This could also be
        done via request to the user, but since we're not really doing a
        RESTful service we do this via the api instead.
        """

        # Check the parameters. Since we only use one parameter we check it
        # here instead of creating a specific parameter parser
        if len(request.params) != 1 or 'dataset' not in request.params:
            return to_jsonp({'error': 'Parameter dataset missing'})

        # Get the dataset we want to check permissions for
        dataset = Dataset.by_name(request.params['dataset'])

        # Return permissions
        return to_jsonp({
                "create":\
                    can.dataset.create() and dataset is None,
                "read":\
                    False if dataset is None else can.dataset.read(dataset),
                "update":\
                    False if dataset is None else can.dataset.update(dataset),
                "delete":\
                    False if dataset is None else can.dataset.delete(dataset)
                })
Beispiel #29
0
 def test_index_hide_private(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.get(url(controller='dataset', action='index', format='json'))
     obj = json.loads(response.body)
     h.assert_equal(len(obj['datasets']), 0)
Beispiel #30
0
    def test_delete(self):
        cra = Dataset.by_name('cra')
        assert len(cra) == 36, len(cra)
        # double-check authz
        response = self.app.post(url(controller='editor',
                                     action='delete', dataset='cra'),
                                 expect_errors=True)
        assert '403' in response.status
        cra = Dataset.by_name('cra')
        assert len(cra) == 36, len(cra)

        response = self.app.post(url(controller='editor',
                                     action='delete', dataset='cra'),
                                 extra_environ={'REMOTE_USER': '******'})
        cra = Dataset.by_name('cra')
        assert cra is None, cra
Beispiel #31
0
def index():
    #page = request.args.get('page')

    q = Dataset.get_all_admin().all()

    returnset = []
    for theobj in q:
        returnset.append(theobj)

    # if len(fields) < 1 and not getsources:
    #     return jsonify(q)

    # returnset = []
    # for obj in q:
    #     tempobj = {}
    #     if len(fields) >0:
    #         for field in fields:
    #             tempobj[field] = getattr(obj, field)
    #     else:
    #         tempobj = obj.as_dict()
    #     if getsources:
    #         tempobj['sources'] = obj.sources.all()
    #     returnset.append(tempobj)

    # TODO: Facets for territories and languages
    # TODO: filters on facet dimensions
    #maybe put the pager back in
    # print q
    # pager = Pager(q)
    return jsonify(returnset, headers={'Cache-Control': 'no-cache'})
Beispiel #32
0
def index():
    #page = request.args.get('page')

    q = Dataset.get_all_admin().all()

    returnset = []
    for theobj in q:
        returnset.append(theobj)

    # if len(fields) < 1 and not getsources:
    #     return jsonify(q)
    
    # returnset = []
    # for obj in q:
    #     tempobj = {} 
    #     if len(fields) >0:
    #         for field in fields:
    #             tempobj[field] = getattr(obj, field)
    #     else:
    #         tempobj = obj.as_dict()
    #     if getsources:
    #         tempobj['sources'] = obj.sources.all()
    #     returnset.append(tempobj) 


    # TODO: Facets for territories and languages
    # TODO: filters on facet dimensions
    #maybe put the pager back in
    # print q
    # pager = Pager(q)
    return jsonify(returnset)
Beispiel #33
0
 def test_index_hide_private(self):
     cra = Dataset.by_name("cra")
     cra.private = True
     db.session.commit()
     response = self.app.get(url(controller="dataset", action="index", format="json"))
     obj = json.loads(response.body)
     h.assert_equal(len(obj["datasets"]), 0)
Beispiel #34
0
def create():
    """
    This takes a json format post with label, name, description
    and creates a private dataset to put sources in
    The json_errors return a json object
    """

    if not require.dataset.create():
        return jsonify(
            {"errors": ["Can not create new dataset.  Permission denied"]})

    try:
        dataset = api_form_data()
        if not dataset.get("dataorg", None):
            return jsonify(
                {"errors": ["You must select the data source organization"]})
        model = {'data': dataset}
        schema = dataset_schema(ValidationState(model))
        data = schema.deserialize(dataset)

        #should have a better place for sluggify
        if (data.get('name', None)):
            tempname = slugify(str(data.get('name')), max_length=50)
        else:
            tempname = slugify(str(data.get('label')), max_length=50)

        if Dataset.by_name(tempname) is not None:
            return jsonify(
                {"errors": ["A dataset with this name already exists "]})

        dataset = Dataset(data=data)
        dataset.managers.append(current_user)
        db.session.add(dataset)

        dataset_source = Source.by_source_name(dataset.name)
        if not dataset_source:
            dataset_source = Source(dataset=dataset, name=dataset.name)
            db.session.add(dataset_source)
        else:
            dataset_source.dataset = dataset
        #creating a new dataset so we have to create a source as well
        db.session.commit()
        return jsonify({"success": True, "dataset": dataset.name})
    except Exception, e:
        ex_type, ex, tb = sys.exc_info()
        print traceback.print_tb(tb)
        return jsonify({"errors": ['Unknown Error has occurred: ' + str(e)]})
Beispiel #35
0
    def index(self):
        # Get all of the datasets available to the account of the logged in
        # or an anonymous user (if c.account is None)
        c.datasets = Dataset.all_by_account(c.account)
        c.territories = DatasetTerritory.dataset_counts(c.datasets)

        c.num_entries = dataset_entries(None)
        return templating.render('home/index.html')
 def setup(self):
     super(TestCompoundDimension, self).setup()
     self.engine = db.engine
     self.meta = db.metadata
     self.meta.bind = self.engine
     self.ds = Dataset(SIMPLE_MODEL)
     self.entity = self.ds['to']
     self.classifier = self.ds['function']
Beispiel #37
0
 def test_view_private(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.get(url(controller='dataset', action='view',
         dataset='cra'), status=403)
     h.assert_false('Country Regional Analysis v2009' in response,
                   "'Country Regional Analysis v2009' not in response!")
     h.assert_false('36 spending entries' in response, "'36 spending entries' not in response!")
Beispiel #38
0
def get_or_create_dataset(model):
    """
    Based on a provided model we get the model (if it doesn't exist we
    create it).
    """

    # Get the dataset by the name provided in the model
    dataset = Dataset.by_name(model['dataset']['name'])

    # If the dataset wasn't found we create it
    if dataset is None:
        dataset = Dataset(model)
        db.session.add(dataset)
        db.session.commit()

    # Log information about the dataset and return it
    log.info("Dataset: %s", dataset.name)
    return dataset
Beispiel #39
0
 def test_templates_update(self):
     response = self.app.post(url(controller='editor',
                                  action='templates_update', dataset='cra'),
                              params={'serp_title': 'BANANA'},
                              extra_environ={'REMOTE_USER': '******'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert cra.serp_title == 'BANANA', cra.serp_title
Beispiel #40
0
 def test_team_update(self):
     response = self.app.post(url(controller='editor',
                                  action='team_update', dataset='cra'),
                              params={},
                              extra_environ={'REMOTE_USER': '******'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert len(cra.managers.all()) == 1, cra.managers
Beispiel #41
0
 def test_core_update_invalid_currency(self):
     response = self.app.post(url(controller='editor', 
         action='core_update', dataset='cra'),
         params={'name': 'cra', 'label': 'Common Rough Act',
                 'description': 'I\'m a banana', 'currency': 'glass pearls'},
         extra_environ={'REMOTE_USER': '******'})
     assert 'not a valid currency' in response.body
     cra = Dataset.by_name('cra')
     assert cra.currency=='GBP', cra.label
Beispiel #42
0
 def test_view_private(self):
     cra = Dataset.by_name("cra")
     cra.private = True
     db.session.commit()
     response = self.app.get(url(controller="dataset", action="view", dataset="cra"), status=403)
     h.assert_false(
         "Country Regional Analysis v2009" in response, "'Country Regional Analysis v2009' not in response!"
     )
     h.assert_false("openspending_browser" in response, "'openspending_browser' not in response!")
Beispiel #43
0
 def test_core_update_invalid_territory(self):
     response = self.app.post(url(controller='editor', 
         action='core_update', dataset='cra'),
         params={'name': 'cra', 'label': 'CRA', 'territories': 'su', 
                 'description': 'I\'m a banana', 'currency': 'GBP'},
         extra_environ={'REMOTE_USER': '******'})
     assert not 'updated' in response.body
     cra = Dataset.by_name('cra')
     assert not 'su' in cra.territories
Beispiel #44
0
    def test_distinct_regions(self):
        b = make_dataset()
        self.dat.add_region("region 1")
        self.dat.add_region("region 2")
        b.add_region("region 1")
        self.dat.save()
        b.save()

        assert Dataset.distinct_regions() == ["region 1", "region 2"]
Beispiel #45
0
 def test_core_update_invalid_label(self):
     response = self.app.post(url(controller='editor', 
         action='core_update', dataset='cra'),
         params={'name': 'cra', 'label': '',
                 'description': 'I\'m a banana', 'currency': 'GBP'},
         extra_environ={'REMOTE_USER': '******'})
     assert 'Required' in response.body
     cra = Dataset.by_name('cra')
     assert cra.label!='', cra.label
Beispiel #46
0
 def test_view_private(self):
     cra = Dataset.by_name('cra')
     cra.private = True
     db.session.commit()
     response = self.app.get(url(controller='dataset', action='view',
         dataset='cra'), status=403)
     h.assert_false('Country Regional Analysis v2009' in response,
                   "'Country Regional Analysis v2009' not in response!")
     h.assert_false('openspending_browser' in response, "'openspending_browser' not in response!")
Beispiel #47
0
    def test_feeds(self):
        # Anonymous user with one public dataset
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True)
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response
        cra = Dataset.by_name('cra')
        cra.private = True
        db.session.add(cra)
        db.session.commit()

        # Anonymous user with one private dataset
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True)
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' not in response

        # Logged in user with one public dataset
        cra.private = False
        db.session.add(cra)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True,
                                extra_environ={'REMOTE_USER': '******'})
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response

        # Logged in user with one private dataset
        cra.private = True
        db.session.add(cra)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True,
                                extra_environ={'REMOTE_USER': '******'})
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' not in response

        # Logged in admin user with one private dataset
        admin_user = h.make_account('admin')
        admin_user.admin = True
        db.session.add(admin_user)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                extra_environ={'REMOTE_USER': '******'})
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response
        assert 'application/xml' in response.content_type

        response = self.app.get(url(controller='dataset', action='index'))
        assert ('<link rel="alternate" type="application/rss+xml" title="'
                'Latest Datasets on OpenSpending" href="/datasets.rss"'
                in response)
Beispiel #48
0
    def test_feeds(self):
        # Anonymous user with one public dataset
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True)
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response
        cra = Dataset.by_name('cra')
        cra.private = True
        db.session.add(cra)
        db.session.commit()

        # Anonymous user with one private dataset
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True)
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' not in response

        # Logged in user with one public dataset
        cra.private = False
        db.session.add(cra)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True,
                                extra_environ={'REMOTE_USER': '******'})
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response

        # Logged in user with one private dataset
        cra.private = True
        db.session.add(cra)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
                                expect_errors=True,
                                extra_environ={'REMOTE_USER': '******'})
        assert 'application/xml' in response.content_type
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' not in response

        # Logged in admin user with one private dataset
        admin_user = h.make_account('admin')
        admin_user.admin = True
        db.session.add(admin_user)
        db.session.commit()
        response = self.app.get(url(controller='dataset', action='feed_rss'),
            extra_environ={'REMOTE_USER': '******'})
        assert '<title>Recently Created Datasets</title>' in response
        assert '<item><title>Country Regional Analysis v2009' in response
        assert 'application/xml' in response.content_type

        response = self.app.get(url(controller='dataset', action='index'))
        assert ('<link rel="alternate" type="application/rss+xml" title="'
            'Latest Datasets on OpenSpending" href="/datasets.rss"' in
            response)
Beispiel #49
0
 def test_core_update(self):
     response = self.app.post(url(controller='editor', 
         action='core_update', dataset='cra'),
         params={'name': 'cra', 'label': 'Common Rough Act',
                 'description': 'I\'m a banana', 'currency': 'EUR',
                 'languages': 'en', 'territories': 'gb'},
         extra_environ={'REMOTE_USER': '******'})
     cra = Dataset.by_name('cra')
     assert cra.label=='Common Rough Act', cra.label
     assert cra.currency=='EUR', cra.currency
Beispiel #50
0
 def test_core_update_invalid_language(self):
     response = self.app.post(url(controller='editor',
         action='core_update', dataset='cra'),
         params={'name': 'cra', 'label': 'CRA', 'languages': 'esperanto',
                 'description': 'I\'m a banana', 'currency': 'GBP',
                 'default_time': 2009},
         extra_environ={'REMOTE_USER': '******'})
     assert not 'updated' in response.body
     cra = Dataset.by_name('cra')
     assert not 'esperanto' in cra.languages
Beispiel #51
0
 def test_templates_update(self):
     response = self.app.post(url(controller='editor',
                                  action='templates_update',
                                  dataset='cra'),
                              params={'serp_title': 'BANANA'},
                              extra_environ={'REMOTE_USER': '******'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert cra.serp_title == 'BANANA', cra.serp_title
Beispiel #52
0
 def test_team_update(self):
     response = self.app.post(url(controller='editor',
                                  action='team_update',
                                  dataset='cra'),
                              params={},
                              extra_environ={'REMOTE_USER': '******'},
                              expect_errors=True)
     assert '200' in response.status, response.status
     cra = Dataset.by_name('cra')
     assert len(cra.managers.all()) == 1, cra.managers
 def test_new_no_apikey(self):
     u = url(controller='api/version2', action='create')
     params = {
         'metadata':
         'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
         'csv_file':
         'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
     }
     response = self.app.post(u, params, expect_errors=True)
     assert "400" in response.status
     assert not Dataset.by_name('openspending-example')
Beispiel #54
0
 def test_dimensions_update_invalid_json(self):
     cra = Dataset.by_name('cra')
     cra.drop()
     cra.init()
     cra.generate()
     response = self.app.post(url(controller='editor', 
         action='dimensions_update', dataset='cra'),
         params={'mapping': 'banana'},
         extra_environ={'REMOTE_USER': '******'},
         expect_errors=True)
     assert '400' in response.status, response.status
Beispiel #55
0
 def list_cubes(self):
     cubes = []
     for dataset in Dataset.all():
         if not len(dataset.mapping):
             continue
         cubes.append({
             #change here too
             'name': dataset.name,
             'label': dataset.label
         })
     return cubes
Beispiel #56
0
 def setup(self):
     super(TestDimensionController, self).setup()
     h.load_fixture('cra')
     h.clean_and_reindex_solr()
     self.cra = Dataset.by_name('cra')
     for dimension in self.cra.dimensions:
         if isinstance(dimension, CompoundDimension) and \
                 dimension.name == 'cofog1':
             members = list(
                 dimension.members(dimension.alias.c.name == '3', limit=1))
             self.member = members.pop()
             break