def load_from_databank(sourcejson, dataproviderjson, dry_run=False, overwrite=True, meta_only=False, file_dir = None):
print "Working on ", sourcejson['fields']['indicator']
dataorg = DataOrg.by_name(dataproviderjson['fields']['title'])
dataorgMeta = {
'description': dataproviderjson['fields']['description'],
'label': dataproviderjson['fields']['title']
}
if not dataorg:
dataorg = DataOrg(dataorgMeta)
db.session.add(dataorg)
#dataorg will update with id here
db.session.commit()
#get or create dataset
dataset = Dataset.by_label(sourcejson['fields']['indicator'])
description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(sourcejson['pk']) + "/"
modelDataset = {'dataset':
{
'label': sourcejson['fields']['indicator'],
'name': sourcejson['fields']['indicator'],
'description': description,
'dataType': sourcejson['fields']['data_type'],
'dataorg_id': dataorg.id
}
}
if not dataset:
#create one
dataset = Dataset(modelDataset['dataset'])
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
db.session.add(dataset)
else:
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
dataset.update(modelDataset['dataset'])
db.session.commit()
systemaccount = Account.by_id(1)
if dataset.source:
try:
print "trying to delete source"
print dataset.source
dataset.source.delete()
except Exception, e:
print "could not delete source", e
def test_browser_for_entity(self):
h.skip_if_stubbed_solr()
from openspending.model import Dataset, Entry
dataset = Dataset(name='testdataset')
Dataset.c.save(dataset, manipulate=True)
dataset_ref_dict = dataset.to_ref_dict()
entity = self._make_one(name="Test Entity", label="Test Entity Label")
entity_ref_dict = entity.to_ref_dict()
entry = {'name': 'Test Entry',
'label': 'Test Entry Label',
'from': entity_ref_dict,
'to': entity_ref_dict,
'amount': 10.0,
'dataset': dataset_ref_dict}
Entry.c.save(entry)
h.clean_and_reindex_solr()
entity_url = url(controller='entity', id=str(entity['_id']),
slug='test-entity-label', action='view')
response = self.app.get(entity_url)
h.assert_equal(response._status, '200 OK')
h.assert_true('<b>1 entries</b> found.<br />' in response)
h.assert_true('entries.json">' in response)
h.assert_true('entries.csv">' in response)
def create(self):
require.dataset.create()
try:
dataset = dict(request.params)
dataset['territories'] = request.params.getall('territories')
dataset['languages'] = request.params.getall('languages')
model = {'dataset': dataset}
schema = dataset_schema(ValidationState(model))
data = schema.deserialize(dataset)
if Dataset.by_name(data['name']) is not None:
raise Invalid(
SchemaNode(String(), name='dataset.name'),
_("A dataset with this identifer already exists!"))
dataset = Dataset({'dataset': data})
dataset.private = True
dataset.managers.append(c.account)
db.session.add(dataset)
db.session.commit()
redirect(
h.url_for(controller='editor',
action='index',
dataset=dataset.name))
except Invalid, i:
errors = i.asdict()
return self.new(errors)
def create(self):
"""
Adds a new dataset dynamically through a POST request
"""
# User must be authenticated so we should have a user object in
# c.account, if not abort with error message
if not c.account:
abort(status_code=400, detail='user not authenticated')
# Check if the params are there ('metadata', 'csv_file')
if len(request.params) != 2:
abort(status_code=400, detail='incorrect number of params')
metadata = request.params['metadata'] \
if 'metadata' in request.params \
else abort(status_code=400, detail='metadata is missing')
csv_file = request.params['csv_file'] \
if 'csv_file' in request.params \
else abort(status_code=400, detail='csv_file is missing')
# We proceed with the dataset
try:
model = json.load(urllib2.urlopen(metadata))
except:
abort(status_code=400, detail='JSON model could not be parsed')
try:
log.info("Validating model")
model = validate_model(model)
except Invalid as i:
log.error("Errors occured during model validation:")
for field, error in i.asdict().items():
log.error("%s: %s", field, error)
abort(status_code=400, detail='Model is not well formed')
dataset = Dataset.by_name(model['dataset']['name'])
if dataset is None:
dataset = Dataset(model)
require.dataset.create()
dataset.managers.append(c.account)
dataset.private = True # Default value
db.session.add(dataset)
else:
require.dataset.update(dataset)
log.info("Dataset: %s", dataset.name)
source = Source(dataset=dataset, creator=c.account, url=csv_file)
log.info(source)
for source_ in dataset.sources:
if source_.url == csv_file:
source = source_
break
db.session.add(source)
db.session.commit()
# Send loading of source into celery queue
load_source.delay(source.id)
return to_jsonp(dataset_apply_links(dataset.as_dict()))
def create(self):
"""
Adds a new dataset dynamically through a POST request
"""
# User must be authenticated so we should have a user object in
# c.account, if not abort with error message
if not c.account:
abort(status_code=400, detail='user not authenticated')
# Check if the params are there ('metadata', 'csv_file')
if len(request.params) != 2:
abort(status_code=400, detail='incorrect number of params')
metadata = request.params['metadata'] \
if 'metadata' in request.params \
else abort(status_code=400, detail='metadata is missing')
csv_file = request.params['csv_file'] \
if 'csv_file' in request.params \
else abort(status_code=400, detail='csv_file is missing')
# We proceed with the dataset
try:
model = json.load(urllib2.urlopen(metadata))
except:
abort(status_code=400, detail='JSON model could not be parsed')
try:
log.info("Validating model")
model = validate_model(model)
except Invalid as i:
log.error("Errors occured during model validation:")
for field, error in i.asdict().items():
log.error("%s: %s", field, error)
abort(status_code=400, detail='Model is not well formed')
dataset = Dataset.by_name(model['dataset']['name'])
if dataset is None:
dataset = Dataset(model)
require.dataset.create()
dataset.managers.append(c.account)
dataset.private = True # Default value
db.session.add(dataset)
else:
require.dataset.update(dataset)
log.info("Dataset: %s", dataset.name)
source = Source(dataset=dataset, creator=c.account, url=csv_file)
log.info(source)
for source_ in dataset.sources:
if source_.url == csv_file:
source = source_
break
db.session.add(source)
db.session.commit()
# Send loading of source into celery queue
load_source.delay(source.id)
return to_jsonp(dataset_apply_links(dataset.as_dict()))
def test_views_update(self):
cra = Dataset.by_name('cra')
views = cra.data['views']
views[0]['label'] = 'Banana'
response = self.app.post(url(controller='editor',
action='views_update', dataset='cra'),
params={'views': json.dumps(views)},
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '200' in response.status, response.status
cra = Dataset.by_name('cra')
assert 'Banana' in repr(cra.data['views'])
def test_views_update(self):
cra = Dataset.by_name('cra')
views = cra.data['views']
views[0]['label'] = 'Banana'
response = self.app.post(url(controller='editor',
action='views_update', dataset='cra'),
params={'views': json.dumps(views)},
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '200' in response.status, response.status
cra = Dataset.by_name('cra')
assert 'Banana' in repr(cra.data['views'])
def test_retract(self):
cra = Dataset.by_name('cra')
assert cra.private is False, cra.private
response = self.app.post(url(controller='editor',
action='retract', dataset='cra'),
extra_environ={'REMOTE_USER': '******'})
cra = Dataset.by_name('cra')
assert cra.private is True, cra.private
response = self.app.post(url(controller='editor',
action='retract', dataset='cra'),
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '400' in response.status, response.status
def test_retract(self):
cra = Dataset.by_name('cra')
assert cra.private is False, cra.private
response = self.app.post(url(controller='editor',
action='retract', dataset='cra'),
extra_environ={'REMOTE_USER': '******'})
cra = Dataset.by_name('cra')
assert cra.private is True, cra.private
response = self.app.post(url(controller='editor',
action='retract', dataset='cra'),
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '400' in response.status, response.status
class TestCompoundDimension(DatabaseTestCase):
def setup(self):
super(TestCompoundDimension, self).setup()
self.engine = db.engine
self.meta = db.metadata
self.meta.bind = self.engine
self.ds = Dataset(SIMPLE_MODEL)
self.entity = self.ds['to']
self.classifier = self.ds['function']
def test_is_compound(self):
h.assert_true(self.entity.is_compound)
def test_basic_properties(self):
assert self.entity.name=='to', self.entity.name
assert self.classifier.name=='function', self.classifier.name
def test_generated_tables(self):
#assert not hasattr(self.entity, 'table'), self.entity
#self.ds.generate()
assert hasattr(self.entity, 'table'), self.entity
assert self.entity.table.name=='test__' + self.entity.taxonomy, self.entity.table.name
assert hasattr(self.entity, 'alias')
assert self.entity.alias.name==self.entity.name, self.entity.alias.name
cols = self.entity.table.c
assert 'id' in cols
assert_raises(KeyError, cols.__getitem__, 'field')
def test_attributes_exist_on_object(self):
assert len(self.entity.attributes)==2, self.entity.attributes
assert_raises(KeyError, self.entity.__getitem__, 'field')
assert self.entity['name'].name=='name'
assert self.entity['name'].datatype=='id'
def test_attributes_exist_on_table(self):
assert hasattr(self.entity, 'table'), self.entity
assert 'name' in self.entity.table.c, self.entity.table.c
assert 'label' in self.entity.table.c, self.entity.table.c
def test_members(self):
self.ds.generate()
self.entity.load(self.ds.bind, {'name': 'one', 'label': 'Label One'})
self.entity.load(self.ds.bind, {'name': 'two', 'label': 'Label Two'})
members = list(self.entity.members())
h.assert_equal(len(members), 2)
members = list(self.entity.members(self.entity.alias.c.name == 'one'))
h.assert_equal(len(members), 1)
def test_publish(self):
cra = Dataset.by_name('cra')
cra.private = True
db.session.commit()
response = self.app.post(url(controller='editor',
action='publish', dataset='cra'),
extra_environ={'REMOTE_USER': '******'})
cra = Dataset.by_name('cra')
assert cra.private is False, cra.private
response = self.app.post(url(controller='editor',
action='publish', dataset='cra'),
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '400' in response.status, response.status
def test_publish(self):
cra = Dataset.by_name('cra')
cra.private = True
db.session.commit()
response = self.app.post(url(controller='editor',
action='publish', dataset='cra'),
extra_environ={'REMOTE_USER': '******'})
cra = Dataset.by_name('cra')
assert cra.private is False, cra.private
response = self.app.post(url(controller='editor',
action='publish', dataset='cra'),
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '400' in response.status, response.status
def csvimport_fixture(name):
model_fp = csvimport_fixture_file(name, 'model.json')
mapping_fp = csvimport_fixture_file(name, 'mapping.json')
model = json.load(model_fp)
if mapping_fp:
model['mapping'] = json.load(mapping_fp)
dataset = Dataset(model)
dataset.generate()
db.session.add(dataset)
data_path = csvimport_fixture_path(name, 'data.csv')
user = make_account()
source = Source(dataset, user, data_path)
db.session.add(source)
db.session.commit()
return source
def csvimport_fixture(name):
model_fp = csvimport_fixture_file(name, 'model.json')
mapping_fp = csvimport_fixture_file(name, 'mapping.json')
model = json.load(model_fp)
if mapping_fp:
model['mapping'] = json.load(mapping_fp)
dataset = Dataset(model)
dataset.generate()
db.session.add(dataset)
data_path = csvimport_fixture_path(name, 'data.csv')
user = h.make_account()
source = Source(dataset, user, data_path)
db.session.add(source)
db.session.commit()
return source
def archive_one(dataset_name, archive_dir):
"""
Find the dataset, create the archive directory and start archiving
"""
# Find the dataset
dataset = Dataset.by_name(dataset_name)
# If no dataset found, exit with error message
if dataset is None:
exit_with_error("Dataset not found. Unable to archive it.")
# If the archive_dir exists we have to ask the user if we should overwrite
if os.path.exists(archive_dir):
# If user doesn't want to write over it we exit
if not get_confirmation("%s exists. Do you want to overwrite?"
% archive_dir):
sys.exit(0)
# If the archive dir is a file we don't do anything
if os.path.isfile(archive_dir):
exit_with_error("Cannot overwrite a file (need a directory).")
# If the archive_dir doesn't exist we create it
else:
try:
os.makedirs(archive_dir)
except OSError:
# If we couldn't create it, we exit with an error message
exit_with_error("Couldn't create archive directory.")
# Archive the model (dataset metadata)
archive_model(dataset, archive_dir)
# Archive the visualisations
archive_visualisations(dataset, archive_dir)
# Download all sources
update(os.path.join(archive_dir, 'sources'), dataset)
def test_drop(self):
cra = Dataset.by_name('cra')
assert len(cra)==36, len(cra)
# double-check authz
response = self.app.post(url(controller='editor',
action='drop', dataset='cra'),
expect_errors=True)
assert '403' in response.status
cra = Dataset.by_name('cra')
assert len(cra)==36, len(cra)
response = self.app.post(url(controller='editor',
action='drop', dataset='cra'),
extra_environ={'REMOTE_USER': '******'})
cra = Dataset.by_name('cra')
assert len(cra)==0, len(cra)
def create_view(self, cls, add_filters, name, label, dimension,
breakdown=None, view_filters={}):
'''\
Create a view. The view will be computed when you call
:meth:`finalize`.
``cls``
A model class (inheriting from :class:`openspending.model.Base`)
``add_filters``
A :term:`mongodb query spec` used as a query to select the
instances of *cls* that will be used for the view.
``name``
A name for the view. This name must be unique for all views
for *cls* in an Open Spending site.
``label``
A label that can be displayed to the user.
``dimensions``
The dimensions that will be used to compute the view
``breakdown``
...
``view_filters``
...
Returns: A :class:`openspending.lib.views.View` object.
'''
log.debug("pre-aggregating view %s on %r where %r",
name, cls, view_filters)
view = View(self.dataset, name, label, dimension,
breakdown, cuts=view_filters)
view.apply_to(cls, add_filters)
view.compute()
Dataset.c.update({'name': self.dataset.name},
{'$set': {'cubes': self.dataset.get('cubes', {})}})
self.dataset = Dataset.by_id(self.dataset.name)
return view
def setup(self):
self.engine = db.engine
self.meta = db.metadata #MetaData()
self.meta.bind = self.engine
self.ds = Dataset(SIMPLE_MODEL)
self.entity = self.ds['to']
self.classifier = self.ds['function']
def setup(self):
super(TestAttributeDimension, self).setup()
self.engine = db.engine
self.meta = db.metadata
self.meta.bind = self.engine
self.ds = Dataset(model_fixture('simple'))
self.field = self.ds['field']
def setup(self):
super(TestAttributeDimension, self).setup()
self.engine = db.engine
self.meta = db.metadata
self.meta.bind = self.engine
self.ds = Dataset(SIMPLE_MODEL)
self.field = self.ds['field']
def test_new_wrong_user(self):
# First we add a Dataset with user 'test_new'
user = Account.by_name('test_new')
assert user.api_key == 'd0610659-627b-4403-8b7f-6e2820ebc95d'
u = url(controller='api/version2', action='create')
params = {
'metadata':
'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
'csv_file':
'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
}
apikey_header = 'apikey {0}'.format(user.api_key)
response = self.app.post(u, params, {'Authorization': apikey_header})
assert "200" in response.status
assert Dataset.by_name('openspending-example')
# After that we try to update the Dataset with user 'test_new2'
user = Account.by_name('test_new2')
assert user.api_key == 'c011c340-8dad-419c-8138-1c6ded86ead5'
u = url(controller='api/version2', action='create')
params = {
'metadata':
'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
'csv_file':
'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
}
apikey_header = 'apikey {0}'.format(user.api_key)
response = self.app.post(u,
params, {'Authorization': apikey_header},
expect_errors=True)
assert '403' in response.status
def test_new_wrong_user(self):
# First we add a Dataset with user 'test_new'
user = Account.by_name('test_new')
assert user.api_key == 'd0610659-627b-4403-8b7f-6e2820ebc95d'
u = url(controller='api/version2', action='create')
params = {
'metadata':
'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
'csv_file':
'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
}
apikey_header = 'apikey {0}'.format(user.api_key)
response = self.app.post(u, params, {'Authorization':apikey_header})
#Dataset.by_name('openspending-example').private = False
assert "200" in response.status
assert Dataset.by_name('openspending-example')
# After that we try to update the Dataset with user 'test_new2'
user = Account.by_name('test_new2')
assert user.api_key == 'c011c340-8dad-419c-8138-1c6ded86ead5'
u = url(controller='api/version2', action='create')
params = {
'metadata':
'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
'csv_file':
'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
}
apikey_header = 'apikey {0}'.format(user.api_key)
response = self.app.post(u, params, {'Authorization':apikey_header},
expect_errors=True)
assert '403' in response.status
def permissions(self):
"""
Check a user's permissions for a given dataset. This could also be
done via request to the user, but since we're not really doing a
RESTful service we do this via the api instead.
"""
# Check the parameters. Since we only use one parameter we check it
# here instead of creating a specific parameter parser
if len(request.params) != 1 or 'dataset' not in request.params:
return to_jsonp({'error': 'Parameter dataset missing'})
# Get the dataset we want to check permissions for
dataset = Dataset.by_name(request.params['dataset'])
# Return permissions
return to_jsonp({
"create":
can.dataset.create() and dataset is None,
"read":
False if dataset is None else can.dataset.read(dataset),
"update":
False if dataset is None else can.dataset.update(dataset),
"delete":
False if dataset is None else can.dataset.delete(dataset)
})
def index(self):
# Get all of the datasets available to the account of the logged in
# or an anonymous user (if c.account is None)
c.datasets = Dataset.all_by_account(c.account)
c.num_entries = dataset_entries(None)
return templating.render('home/index.html')
def remove_dataset(dataset_name):
log.warn("Dropping dataset '%s'", dataset_name)
from openspending.model import Dataset, meta as db
dataset = Dataset.by_name(dataset_name)
dataset.drop()
db.session.delete(dataset)
db.session.commit()
def archive_one(dataset_name, archive_dir):
"""
Find the dataset, create the archive directory and start archiving
"""
# Find the dataset
dataset = Dataset.by_name(dataset_name)
# If no dataset found, exit with error message
if dataset is None:
exit_with_error("Dataset not found. Unable to archive it.")
# If the archive_dir exists we have to ask the user if we should overwrite
if os.path.exists(archive_dir):
# If user doesn't want to write over it we exit
if not get_confirmation(
"%s exists. Do you want to overwrite?" % archive_dir):
sys.exit(0)
# If the archive dir is a file we don't do anything
if os.path.isfile(archive_dir):
exit_with_error("Cannot overwrite a file (need a directory).")
# If the archive_dir doesn't exist we create it
else:
try:
os.makedirs(archive_dir)
except OSError:
# If we couldn't create it, we exit with an error message
exit_with_error("Couldn't create archive directory.")
# Archive the model (dataset metadata)
archive_model(dataset, archive_dir)
# Archive the visualisations
archive_visualisations(dataset, archive_dir)
# Download all sources
update(os.path.join(archive_dir, 'sources'), dataset)
def test_index_hide_private(self):
cra = Dataset.by_name('cra')
cra.private = True
db.session.commit()
response = self.app.get(url(controller='dataset', action='index', format='json'))
obj = json.loads(response.body)
h.assert_equal(len(obj), 0)
def permissions(self):
"""
Check a user's permissions for a given dataset. This could also be
done via request to the user, but since we're not really doing a
RESTful service we do this via the api instead.
"""
# Check the parameters. Since we only use one parameter we check it
# here instead of creating a specific parameter parser
if len(request.params) != 1 or 'dataset' not in request.params:
return to_jsonp({'error': 'Parameter dataset missing'})
# Get the dataset we want to check permissions for
dataset = Dataset.by_name(request.params['dataset'])
# Return permissions
return to_jsonp({
"create":\
can.dataset.create() and dataset is None,
"read":\
False if dataset is None else can.dataset.read(dataset),
"update":\
False if dataset is None else can.dataset.update(dataset),
"delete":\
False if dataset is None else can.dataset.delete(dataset)
})
def test_index_hide_private(self):
cra = Dataset.by_name('cra')
cra.private = True
db.session.commit()
response = self.app.get(url(controller='dataset', action='index', format='json'))
obj = json.loads(response.body)
h.assert_equal(len(obj['datasets']), 0)
def test_delete(self):
cra = Dataset.by_name('cra')
assert len(cra) == 36, len(cra)
# double-check authz
response = self.app.post(url(controller='editor',
action='delete', dataset='cra'),
expect_errors=True)
assert '403' in response.status
cra = Dataset.by_name('cra')
assert len(cra) == 36, len(cra)
response = self.app.post(url(controller='editor',
action='delete', dataset='cra'),
extra_environ={'REMOTE_USER': '******'})
cra = Dataset.by_name('cra')
assert cra is None, cra
def index():
#page = request.args.get('page')
q = Dataset.get_all_admin().all()
returnset = []
for theobj in q:
returnset.append(theobj)
# if len(fields) < 1 and not getsources:
# return jsonify(q)
# returnset = []
# for obj in q:
# tempobj = {}
# if len(fields) >0:
# for field in fields:
# tempobj[field] = getattr(obj, field)
# else:
# tempobj = obj.as_dict()
# if getsources:
# tempobj['sources'] = obj.sources.all()
# returnset.append(tempobj)
# TODO: Facets for territories and languages
# TODO: filters on facet dimensions
#maybe put the pager back in
# print q
# pager = Pager(q)
return jsonify(returnset, headers={'Cache-Control': 'no-cache'})
def index():
#page = request.args.get('page')
q = Dataset.get_all_admin().all()
returnset = []
for theobj in q:
returnset.append(theobj)
# if len(fields) < 1 and not getsources:
# return jsonify(q)
# returnset = []
# for obj in q:
# tempobj = {}
# if len(fields) >0:
# for field in fields:
# tempobj[field] = getattr(obj, field)
# else:
# tempobj = obj.as_dict()
# if getsources:
# tempobj['sources'] = obj.sources.all()
# returnset.append(tempobj)
# TODO: Facets for territories and languages
# TODO: filters on facet dimensions
#maybe put the pager back in
# print q
# pager = Pager(q)
return jsonify(returnset)
def test_index_hide_private(self):
cra = Dataset.by_name("cra")
cra.private = True
db.session.commit()
response = self.app.get(url(controller="dataset", action="index", format="json"))
obj = json.loads(response.body)
h.assert_equal(len(obj["datasets"]), 0)
def create():
"""
This takes a json format post with label, name, description
and creates a private dataset to put sources in
The json_errors return a json object
"""
if not require.dataset.create():
return jsonify(
{"errors": ["Can not create new dataset. Permission denied"]})
try:
dataset = api_form_data()
if not dataset.get("dataorg", None):
return jsonify(
{"errors": ["You must select the data source organization"]})
model = {'data': dataset}
schema = dataset_schema(ValidationState(model))
data = schema.deserialize(dataset)
#should have a better place for sluggify
if (data.get('name', None)):
tempname = slugify(str(data.get('name')), max_length=50)
else:
tempname = slugify(str(data.get('label')), max_length=50)
if Dataset.by_name(tempname) is not None:
return jsonify(
{"errors": ["A dataset with this name already exists "]})
dataset = Dataset(data=data)
dataset.managers.append(current_user)
db.session.add(dataset)
dataset_source = Source.by_source_name(dataset.name)
if not dataset_source:
dataset_source = Source(dataset=dataset, name=dataset.name)
db.session.add(dataset_source)
else:
dataset_source.dataset = dataset
#creating a new dataset so we have to create a source as well
db.session.commit()
return jsonify({"success": True, "dataset": dataset.name})
except Exception, e:
ex_type, ex, tb = sys.exc_info()
print traceback.print_tb(tb)
return jsonify({"errors": ['Unknown Error has occurred: ' + str(e)]})
def index(self):
# Get all of the datasets available to the account of the logged in
# or an anonymous user (if c.account is None)
c.datasets = Dataset.all_by_account(c.account)
c.territories = DatasetTerritory.dataset_counts(c.datasets)
c.num_entries = dataset_entries(None)
return templating.render('home/index.html')
def setup(self):
super(TestCompoundDimension, self).setup()
self.engine = db.engine
self.meta = db.metadata
self.meta.bind = self.engine
self.ds = Dataset(SIMPLE_MODEL)
self.entity = self.ds['to']
self.classifier = self.ds['function']
def test_view_private(self):
cra = Dataset.by_name('cra')
cra.private = True
db.session.commit()
response = self.app.get(url(controller='dataset', action='view',
dataset='cra'), status=403)
h.assert_false('Country Regional Analysis v2009' in response,
"'Country Regional Analysis v2009' not in response!")
h.assert_false('36 spending entries' in response, "'36 spending entries' not in response!")
def get_or_create_dataset(model):
"""
Based on a provided model we get the model (if it doesn't exist we
create it).
"""
# Get the dataset by the name provided in the model
dataset = Dataset.by_name(model['dataset']['name'])
# If the dataset wasn't found we create it
if dataset is None:
dataset = Dataset(model)
db.session.add(dataset)
db.session.commit()
# Log information about the dataset and return it
log.info("Dataset: %s", dataset.name)
return dataset
def test_templates_update(self):
response = self.app.post(url(controller='editor',
action='templates_update', dataset='cra'),
params={'serp_title': 'BANANA'},
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '200' in response.status, response.status
cra = Dataset.by_name('cra')
assert cra.serp_title == 'BANANA', cra.serp_title
def test_team_update(self):
response = self.app.post(url(controller='editor',
action='team_update', dataset='cra'),
params={},
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '200' in response.status, response.status
cra = Dataset.by_name('cra')
assert len(cra.managers.all()) == 1, cra.managers
def test_core_update_invalid_currency(self):
response = self.app.post(url(controller='editor',
action='core_update', dataset='cra'),
params={'name': 'cra', 'label': 'Common Rough Act',
'description': 'I\'m a banana', 'currency': 'glass pearls'},
extra_environ={'REMOTE_USER': '******'})
assert 'not a valid currency' in response.body
cra = Dataset.by_name('cra')
assert cra.currency=='GBP', cra.label
def test_view_private(self):
cra = Dataset.by_name("cra")
cra.private = True
db.session.commit()
response = self.app.get(url(controller="dataset", action="view", dataset="cra"), status=403)
h.assert_false(
"Country Regional Analysis v2009" in response, "'Country Regional Analysis v2009' not in response!"
)
h.assert_false("openspending_browser" in response, "'openspending_browser' not in response!")
def test_core_update_invalid_territory(self):
response = self.app.post(url(controller='editor',
action='core_update', dataset='cra'),
params={'name': 'cra', 'label': 'CRA', 'territories': 'su',
'description': 'I\'m a banana', 'currency': 'GBP'},
extra_environ={'REMOTE_USER': '******'})
assert not 'updated' in response.body
cra = Dataset.by_name('cra')
assert not 'su' in cra.territories
def test_distinct_regions(self):
b = make_dataset()
self.dat.add_region("region 1")
self.dat.add_region("region 2")
b.add_region("region 1")
self.dat.save()
b.save()
assert Dataset.distinct_regions() == ["region 1", "region 2"]
def test_core_update_invalid_label(self):
response = self.app.post(url(controller='editor',
action='core_update', dataset='cra'),
params={'name': 'cra', 'label': '',
'description': 'I\'m a banana', 'currency': 'GBP'},
extra_environ={'REMOTE_USER': '******'})
assert 'Required' in response.body
cra = Dataset.by_name('cra')
assert cra.label!='', cra.label
def test_view_private(self):
cra = Dataset.by_name('cra')
cra.private = True
db.session.commit()
response = self.app.get(url(controller='dataset', action='view',
dataset='cra'), status=403)
h.assert_false('Country Regional Analysis v2009' in response,
"'Country Regional Analysis v2009' not in response!")
h.assert_false('openspending_browser' in response, "'openspending_browser' not in response!")
def test_feeds(self):
# Anonymous user with one public dataset
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True)
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' in response
cra = Dataset.by_name('cra')
cra.private = True
db.session.add(cra)
db.session.commit()
# Anonymous user with one private dataset
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True)
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' not in response
# Logged in user with one public dataset
cra.private = False
db.session.add(cra)
db.session.commit()
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True,
extra_environ={'REMOTE_USER': '******'})
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' in response
# Logged in user with one private dataset
cra.private = True
db.session.add(cra)
db.session.commit()
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True,
extra_environ={'REMOTE_USER': '******'})
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' not in response
# Logged in admin user with one private dataset
admin_user = h.make_account('admin')
admin_user.admin = True
db.session.add(admin_user)
db.session.commit()
response = self.app.get(url(controller='dataset', action='feed_rss'),
extra_environ={'REMOTE_USER': '******'})
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' in response
assert 'application/xml' in response.content_type
response = self.app.get(url(controller='dataset', action='index'))
assert ('<link rel="alternate" type="application/rss+xml" title="'
'Latest Datasets on OpenSpending" href="/datasets.rss"'
in response)
def test_feeds(self):
# Anonymous user with one public dataset
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True)
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' in response
cra = Dataset.by_name('cra')
cra.private = True
db.session.add(cra)
db.session.commit()
# Anonymous user with one private dataset
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True)
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' not in response
# Logged in user with one public dataset
cra.private = False
db.session.add(cra)
db.session.commit()
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True,
extra_environ={'REMOTE_USER': '******'})
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' in response
# Logged in user with one private dataset
cra.private = True
db.session.add(cra)
db.session.commit()
response = self.app.get(url(controller='dataset', action='feed_rss'),
expect_errors=True,
extra_environ={'REMOTE_USER': '******'})
assert 'application/xml' in response.content_type
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' not in response
# Logged in admin user with one private dataset
admin_user = h.make_account('admin')
admin_user.admin = True
db.session.add(admin_user)
db.session.commit()
response = self.app.get(url(controller='dataset', action='feed_rss'),
extra_environ={'REMOTE_USER': '******'})
assert '<title>Recently Created Datasets</title>' in response
assert '<item><title>Country Regional Analysis v2009' in response
assert 'application/xml' in response.content_type
response = self.app.get(url(controller='dataset', action='index'))
assert ('<link rel="alternate" type="application/rss+xml" title="'
'Latest Datasets on OpenSpending" href="/datasets.rss"' in
response)
def test_core_update(self):
response = self.app.post(url(controller='editor',
action='core_update', dataset='cra'),
params={'name': 'cra', 'label': 'Common Rough Act',
'description': 'I\'m a banana', 'currency': 'EUR',
'languages': 'en', 'territories': 'gb'},
extra_environ={'REMOTE_USER': '******'})
cra = Dataset.by_name('cra')
assert cra.label=='Common Rough Act', cra.label
assert cra.currency=='EUR', cra.currency
def test_core_update_invalid_language(self):
response = self.app.post(url(controller='editor',
action='core_update', dataset='cra'),
params={'name': 'cra', 'label': 'CRA', 'languages': 'esperanto',
'description': 'I\'m a banana', 'currency': 'GBP',
'default_time': 2009},
extra_environ={'REMOTE_USER': '******'})
assert not 'updated' in response.body
cra = Dataset.by_name('cra')
assert not 'esperanto' in cra.languages
def test_templates_update(self):
response = self.app.post(url(controller='editor',
action='templates_update',
dataset='cra'),
params={'serp_title': 'BANANA'},
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '200' in response.status, response.status
cra = Dataset.by_name('cra')
assert cra.serp_title == 'BANANA', cra.serp_title
def test_team_update(self):
response = self.app.post(url(controller='editor',
action='team_update',
dataset='cra'),
params={},
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '200' in response.status, response.status
cra = Dataset.by_name('cra')
assert len(cra.managers.all()) == 1, cra.managers
def test_new_no_apikey(self):
u = url(controller='api/version2', action='create')
params = {
'metadata':
'https://dl.dropbox.com/u/3250791/sample-openspending-model.json',
'csv_file':
'http://mk.ucant.org/info/data/sample-openspending-dataset.csv'
}
response = self.app.post(u, params, expect_errors=True)
assert "400" in response.status
assert not Dataset.by_name('openspending-example')
def test_dimensions_update_invalid_json(self):
cra = Dataset.by_name('cra')
cra.drop()
cra.init()
cra.generate()
response = self.app.post(url(controller='editor',
action='dimensions_update', dataset='cra'),
params={'mapping': 'banana'},
extra_environ={'REMOTE_USER': '******'},
expect_errors=True)
assert '400' in response.status, response.status
def list_cubes(self):
cubes = []
for dataset in Dataset.all():
if not len(dataset.mapping):
continue
cubes.append({
#change here too
'name': dataset.name,
'label': dataset.label
})
return cubes
def setup(self):
super(TestDimensionController, self).setup()
h.load_fixture('cra')
h.clean_and_reindex_solr()
self.cra = Dataset.by_name('cra')
for dimension in self.cra.dimensions:
if isinstance(dimension, CompoundDimension) and \
dimension.name == 'cofog1':
members = list(
dimension.members(dimension.alias.c.name == '3', limit=1))
self.member = members.pop()
break