def setup(self):
super(TestAttributeDimension, self).setup()
self.engine = db.engine
self.meta = db.metadata
self.meta.bind = self.engine
self.ds = Dataset(model_fixture('simple'))
self.field = self.ds['field']
def create(self):
require.dataset.create()
try:
dataset = dict(request.params)
dataset['territories'] = request.params.getall('territories')
dataset['languages'] = request.params.getall('languages')
model = {'dataset': dataset}
schema = dataset_schema(ValidationState(model))
data = schema.deserialize(dataset)
if Dataset.by_name(data['name']) is not None:
raise Invalid(
SchemaNode(String(), name='dataset.name'),
_("A dataset with this identifer already exists!"))
dataset = Dataset({'dataset': data})
dataset.private = True
dataset.managers.append(c.account)
db.session.add(dataset)
db.session.commit()
redirect(
h.url_for(controller='editor',
action='index',
dataset=dataset.name))
except Invalid, i:
errors = i.asdict()
return self.new(errors)
def setup(self):
self.engine = db.engine
self.meta = db.metadata #MetaData()
self.meta.bind = self.engine
self.ds = Dataset(SIMPLE_MODEL)
self.entity = self.ds['to']
self.classifier = self.ds['function']
def setup(self):
super(TestAttributeDimension, self).setup()
self.engine = db.engine
self.meta = db.metadata
self.meta.bind = self.engine
self.ds = Dataset(SIMPLE_MODEL)
self.field = self.ds['field']
def create(self):
"""
Adds a new dataset dynamically through a POST request
"""
# User must be authenticated so we should have a user object in
# c.account, if not abort with error message
if not c.account:
abort(status_code=400, detail='user not authenticated')
# Check if the params are there ('metadata', 'csv_file')
if len(request.params) != 2:
abort(status_code=400, detail='incorrect number of params')
metadata = request.params['metadata'] \
if 'metadata' in request.params \
else abort(status_code=400, detail='metadata is missing')
csv_file = request.params['csv_file'] \
if 'csv_file' in request.params \
else abort(status_code=400, detail='csv_file is missing')
# We proceed with the dataset
try:
model = json.load(urllib2.urlopen(metadata))
except:
abort(status_code=400, detail='JSON model could not be parsed')
try:
log.info("Validating model")
model = validate_model(model)
except Invalid as i:
log.error("Errors occured during model validation:")
for field, error in i.asdict().items():
log.error("%s: %s", field, error)
abort(status_code=400, detail='Model is not well formed')
dataset = Dataset.by_name(model['dataset']['name'])
if dataset is None:
dataset = Dataset(model)
require.dataset.create()
dataset.managers.append(c.account)
dataset.private = True # Default value
db.session.add(dataset)
else:
require.dataset.update(dataset)
log.info("Dataset: %s", dataset.name)
source = Source(dataset=dataset, creator=c.account, url=csv_file)
log.info(source)
for source_ in dataset.sources:
if source_.url == csv_file:
source = source_
break
db.session.add(source)
db.session.commit()
# Send loading of source into celery queue
load_source.delay(source.id)
return to_jsonp(dataset_apply_links(dataset.as_dict()))
def csvimport_fixture(name):
model_fp = csvimport_fixture_file(name, 'model.json')
mapping_fp = csvimport_fixture_file(name, 'mapping.json')
model = json.load(model_fp)
if mapping_fp:
model['mapping'] = json.load(mapping_fp)
dataset = Dataset(model)
dataset.generate()
db.session.add(dataset)
data_path = csvimport_fixture_path(name, 'data.csv')
user = make_account()
source = Source(dataset, user, data_path)
db.session.add(source)
db.session.commit()
return source
def create():
"""
This takes a json format post with label, name, description
and creates a private dataset to put sources in
The json_errors return a json object
"""
if not require.dataset.create():
return jsonify(
{"errors": ["Can not create new dataset. Permission denied"]})
try:
dataset = api_form_data()
if not dataset.get("dataorg", None):
return jsonify(
{"errors": ["You must select the data source organization"]})
model = {'data': dataset}
schema = dataset_schema(ValidationState(model))
data = schema.deserialize(dataset)
#should have a better place for sluggify
if (data.get('name', None)):
tempname = slugify(str(data.get('name')), max_length=50)
else:
tempname = slugify(str(data.get('label')), max_length=50)
if Dataset.by_name(tempname) is not None:
return jsonify(
{"errors": ["A dataset with this name already exists "]})
dataset = Dataset(data=data)
dataset.managers.append(current_user)
db.session.add(dataset)
dataset_source = Source.by_source_name(dataset.name)
if not dataset_source:
dataset_source = Source(dataset=dataset, name=dataset.name)
db.session.add(dataset_source)
else:
dataset_source.dataset = dataset
#creating a new dataset so we have to create a source as well
db.session.commit()
return jsonify({"success": True, "dataset": dataset.name})
except Exception, e:
ex_type, ex, tb = sys.exc_info()
print traceback.print_tb(tb)
return jsonify({"errors": ['Unknown Error has occurred: ' + str(e)]})
def get_or_create_dataset(model):
"""
Based on a provided model we get the model (if it doesn't exist we
create it).
"""
# Get the dataset by the name provided in the model
dataset = Dataset.by_name(model['dataset']['name'])
# If the dataset wasn't found we create it
if dataset is None:
dataset = Dataset(model)
db.session.add(dataset)
db.session.commit()
# Log information about the dataset and return it
log.info("Dataset: %s", dataset.name)
return dataset
def load_fixture(name, manager=None):
"""
Load fixture data into the database.
"""
model = model_fixture(name)
dataset = Dataset(model)
dataset.updated_at = datetime.utcnow()
if manager is not None:
dataset.managers.append(manager)
db.session.add(dataset)
db.session.commit()
dataset.generate()
data = data_fixture(name)
reader = csv.DictReader(data)
for row in reader:
entry = convert_types(model['mapping'], row)
dataset.load(entry)
data.close()
dataset.commit()
return dataset
def load_fixture(name, manager=None):
"""
Load fixture data into the database.
"""
from openspending.validation.data import convert_types
fh = open(fixture_path('%s.js' % name), 'r')
data = json.load(fh)
fh.close()
dataset = Dataset(data)
if manager is not None:
dataset.managers.append(manager)
db.session.add(dataset)
db.session.commit()
dataset.generate()
fh = open(fixture_path('%s.csv' % name), 'r')
reader = csv.DictReader(fh)
for row in reader:
entry = convert_types(data['mapping'], row)
dataset.load(entry)
fh.close()
dataset.commit()
return dataset
def setup(self):
super(TestDatasetLoad, self).setup()
self.ds = Dataset(SIMPLE_MODEL)
self.ds.generate()
self.engine = db.engine
def setup(self):
super(TestDataset, self).setup()
self.ds = Dataset(SIMPLE_MODEL)
def setup(self):
super(TestDatasetLoad, self).setup()
self.ds = Dataset(model_fixture('simple'))
self.ds.generate()
self.engine = db.engine
def setup(self):
super(TestDataset, self).setup()
self.model = model_fixture('simple')
self.ds = Dataset(self.model)
print("You must provide --model!", file=sys.stderr)
return 1
model = json_of_url(args.model)
try:
log.info("Validating model")
model = validate_model(model)
except Invalid, i:
log.error("Errors occured during model validation:")
for field, error in i.asdict().items():
log.error("%s: %s", field, error)
return 1
dataset = Dataset.by_name(model['dataset']['name'])
if dataset is None:
dataset = Dataset(model)
db.session.add(dataset)
log.info("Dataset: %s", dataset.name)
source = Source(dataset, shell_account(), csv_data_url)
for source_ in dataset.sources:
if source_.url == csv_data_url:
source = source_
break
db.session.add(source)
db.session.commit()
dataset.generate()
importer = CSVImporter(source)
importer.run(**vars(args))
return 0
def load_from_databank(sourcejson,
dataproviderjson,
dry_run=False,
overwrite=True,
meta_only=False,
file_dir=None):
print "Working on ", sourcejson['fields']['indicator']
dataorg = DataOrg.by_name(dataproviderjson['fields']['title'])
dataorgMeta = {
'description': dataproviderjson['fields']['description'],
'label': dataproviderjson['fields']['title']
}
if not dataorg:
dataorg = DataOrg(dataorgMeta)
db.session.add(dataorg)
#dataorg will update with id here
db.session.commit()
#get or create dataset
dataset = Dataset.by_label(sourcejson['fields']['indicator'])
description = "http://databank.edip-maps.net/admin/etldata/dataconnection/" + str(
sourcejson['pk']) + "/"
modelDataset = {
'dataset': {
'label': sourcejson['fields']['indicator'],
'name': sourcejson['fields']['indicator'],
'description': description,
'dataType': sourcejson['fields']['data_type'],
'dataorg_id': dataorg.id
}
}
if not dataset:
#create one
dataset = Dataset(modelDataset['dataset'])
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
db.session.add(dataset)
else:
#dataset.ORoperations = dataproviderjson['fields'].get('ORoperations', {})
#dataset.data = dataproviderjson['fields'].get('mapping',{})
dataset.update(modelDataset['dataset'])
db.session.commit()
systemaccount = Account.by_id(1)
if dataset.source:
try:
print "trying to delete source"
print dataset.source
dataset.source.delete()
except Exception, e:
print "could not delete source", e
