class Source(db.Model):
__tablename__ = 'source'
id = db.Column(db.Integer, primary_key=True)
url = db.Column(db.Unicode)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)
analysis = db.Column(JSONType, default=dict)
dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
dataset = db.relationship(Dataset,
backref=db.backref(
'sources',
lazy='dynamic',
order_by='Source.created_at.desc()'))
creator_id = db.Column(db.Integer, db.ForeignKey('account.id'))
creator = db.relationship(Account,
backref=db.backref('sources', lazy='dynamic'))
def __init__(self, dataset, creator, url):
self.dataset = dataset
self.creator = creator
self.url = url
@property
def name(self):
if len(self.url) > 55:
return self.url[:15] + " .. " + self.url[len(self.url) - 25:]
return self.url
@property
def loadable(self):
if not len(self.dataset.mapping):
return False
if 'error' in self.analysis:
return False
return True
def __repr__(self):
return "<Source(%s,%s)>" % (self.dataset.name, self.name)
@classmethod
def by_id(cls, id):
return db.session.query(cls).filter_by(id=id).first()
def as_dict(self):
return {
"id": self.id,
"url": self.url,
"dataset": self.dataset.name,
"created_at": self.created_at
}
class LogRecord(db.Model):
__tablename__ = 'log_record'
CATEGORY_SYSTEM = 'system'
CATEGORY_MODEL = 'model'
CATEGORY_DATA = 'data'
id = db.Column(db.Integer, primary_key=True)
run_id = db.Column(db.Integer, db.ForeignKey('run.id'))
category = db.Column(db.Unicode)
level = db.Column(db.Unicode)
message = db.Column(db.Unicode)
error = db.Column(db.Unicode)
timestamp = db.Column(db.DateTime, default=datetime.utcnow)
row = db.Column(db.Integer)
attribute = db.Column(db.Unicode)
column = db.Column(db.Unicode)
data_type = db.Column(db.Unicode)
value = db.Column(db.Unicode)
run = db.relationship(Run, backref=db.backref('records', lazy='dynamic'))
def __init__(self, run, category, level, message):
self.run = run
self.category = category
self.level = level
self.message = message
@classmethod
def by_id(cls, id):
return db.session.query(cls).filter_by(id=id).first()
class Run(db.Model):
""" A run is a generic grouping object for background operations
that perform logging to the frontend. """
__tablename__ = 'run'
STATUS_RUNNING = 'running'
STATUS_COMPLETE = 'complete'
STATUS_FAILED = 'failed'
id = db.Column(db.Integer, primary_key=True)
operation = db.Column(db.Unicode(2000))
status = db.Column(db.Unicode(2000))
time_start = db.Column(db.DateTime, default=datetime.utcnow)
time_end = db.Column(db.DateTime)
dataset_id = db.Column(db.Integer,
db.ForeignKey('dataset.id'),
nullable=True)
source_id = db.Column(db.Integer,
db.ForeignKey('source.id'),
nullable=True)
dataset = db.relationship(Dataset,
backref=db.backref(
'runs',
order_by='Run.time_start.desc()',
lazy='dynamic'))
source = db.relationship(Source,
backref=db.backref(
'runs',
order_by='Run.time_start.desc()',
lazy='dynamic'))
def __init__(self, operation, status, dataset, source):
self.operation = operation
self.status = status
self.dataset = dataset
self.source = source
@classmethod
def by_id(cls, id):
return db.session.query(cls).filter_by(id=id).first()
def __repr__(self):
return "<Run(%s,%s)>" % (self.source.id, self.id)
class DatasetCountry(db.Model):
__tablename__ = 'dataset_country'
id = db.Column(db.Integer, primary_key=True)
code = db.Column(db.Unicode)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)
dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
dataset = db.relationship(Dataset,
backref=db.backref('countries', lazy='dynamic'))
def __init__(self, dataset, code):
self.dataset = dataset
self.code = code
class DatasetLanguage(db.Model, DatasetFacetMixin):
__tablename__ = 'dataset_language'
id = db.Column(db.Integer, primary_key=True)
code = db.Column(db.Unicode)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)
dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
dataset = db.relationship(Dataset,
backref=db.backref('_languages', lazy=False))
def __init__(self, code):
# self.dataset = dataset
self.code = code
class Badge(db.Model):
"""
This model allows marking datasets with various badges.
Examples could be "World Bank" - data verified by the World bank.
Each badge has a name, a representative image and a description.
Also stored for historical reasons are badge creator, creation time
and modification date.
"""
__tablename__ = 'badge'
id = db.Column(db.Integer, primary_key=True)
# Primary information for this badge
label = db.Column(db.Unicode)
image = db.Column(db.Unicode)
description = db.Column(db.Unicode)
# Define relationship with datasets via the associate table
datasets = db.relationship("Dataset",
secondary=badges_on_datasets,
backref=db.backref('badges', lazy='dynamic'))
# Creator (and relationship)
creator_id = db.Column(db.Integer, db.ForeignKey('account.id'))
creator = db.relationship(Account,
backref=db.backref('badge_creations',
lazy='dynamic'))
# Timestamps
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime,
default=datetime.utcnow,
onupdate=datetime.utcnow)
def __init__(self, label, image, description, creator):
"""
Initialize a badge object.
Badge label should be a representative title for the badge
Image should be a small, representative image for the badge
Description describes the purpose of the badge in more detail
Creator is the user who created the badge.
"""
self.label = label
self.image = image
self.description = description
self.creator = creator
def __repr__(self):
return "<Badge(%s)>" % self.label
@classmethod
def by_id(cls, id):
"""
Find one badge given the id
"""
return db.session.query(cls).filter_by(id=id).first()
@classmethod
def all(cls):
"""
Find all badges
"""
return db.session.query(cls)
def as_dict(self, short=False):
"""
A dictionary representation of the badge. This can return a long
version containing all interesting fields or a short version containing
only id, label and image.
"""
badge = {
"id": self.id,
"label": self.label,
"image": self.image,
}
if not short:
badge.update({
"description": self.description,
"datasets": [ds.name for ds in self.datasets],
"created_at": self.created_at
})
return badge
from datetime import datetime
from openspending.model import Account, meta as db
# Badges and dataset share a many to many relationship
# therefore we need to create an associate table
badges_on_datasets = db.Table(
'badges_on_datasets', db.metadata,
db.Column('badge_id', db.Integer, db.ForeignKey('badge.id')),
db.Column('dataset_id', db.Integer, db.ForeignKey('dataset.id')))
class Badge(db.Model):
"""
This model allows marking datasets with various badges.
Examples could be "World Bank" - data verified by the World bank.
Each badge has a name, a representative image and a description.
Also stored for historical reasons are badge creator, creation time
and modification date.
"""
__tablename__ = 'badge'
id = db.Column(db.Integer, primary_key=True)
# Primary information for this badge
label = db.Column(db.Unicode)
image = db.Column(db.Unicode)
description = db.Column(db.Unicode)
# Define relationship with datasets via the associate table
from openspending.model import meta as db
from openspending.model.dataset import Dataset
REGISTER_NAME_RE = r"^[a-zA-Z0-9_\-]{3,255}$"
def make_uuid():
return unicode(uuid.uuid4())
account_dataset_table = db.Table(
'account_dataset', db.metadata,
db.Column('dataset_id',
db.Integer,
db.ForeignKey('dataset.id'),
primary_key=True),
db.Column('account_id',
db.Integer,
db.ForeignKey('account.id'),
primary_key=True))
class Account(db.Model):
__tablename__ = 'account'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Unicode(255), unique=True)
fullname = db.Column(db.Unicode(2000))
email = db.Column(db.Unicode(2000))
password = db.Column(db.Unicode(2000))
class View(db.Model):
""" A view stores a specific configuration of a visualisation widget. """
__tablename__ = 'view'
id = db.Column(db.Integer, primary_key=True)
widget = db.Column(db.Unicode(2000))
name = db.Column(db.Unicode(2000))
label = db.Column(db.Unicode(2000))
description = db.Column(db.Unicode())
state = db.Column(MutableDict.as_mutable(JSONType), default=dict)
public = db.Column(db.Boolean, default=False)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)
dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
account_id = db.Column(db.Integer,
db.ForeignKey('account.id'),
nullable=True)
dataset = db.relationship(Dataset,
backref=db.backref(
'views',
cascade='all,delete,delete-orphan',
lazy='dynamic'))
account = db.relationship(Account,
backref=db.backref(
'views',
cascade='all,delete,delete-orphan',
lazy='dynamic'))
def __init__(self):
pass
@classmethod
def by_id(cls, id):
return db.session.query(cls).filter_by(id=id).first()
@classmethod
def by_name(cls, dataset, name):
q = db.session.query(cls).filter_by(name=name)
return q.filter_by(dataset=dataset).first()
@classmethod
def all_by_dataset(cls, dataset):
return db.session.query(cls).filter_by(dataset=dataset)
def as_dict(self):
return {
'id': self.id,
'widget': self.widget,
'name': self.name,
'label': self.label,
'description': self.description,
'state': self.state,
'public': self.public,
'dataset': self.dataset.name,
'account': self.account.name if self.account else None
}
def __repr__(self):
return "<View(%s,%s)>" % (self.dataset.name, self.name)
class Run(db.Model):
""" A run is a generic grouping object for background operations
that perform logging to the frontend. """
__tablename__ = 'run'
# Status values
STATUS_RUNNING = 'running'
STATUS_COMPLETE = 'complete'
STATUS_FAILED = 'failed'
STATUS_REMOVED = 'removed'
# Operation values for database, two operations possible
OPERATION_SAMPLE = 'sample'
OPERATION_IMPORT = 'import'
id = db.Column(db.Integer, primary_key=True)
operation = db.Column(db.Unicode(2000))
status = db.Column(db.Unicode(2000))
time_start = db.Column(db.DateTime, default=datetime.utcnow)
time_end = db.Column(db.DateTime)
dataset_id = db.Column(db.Integer,
db.ForeignKey('dataset.id'),
nullable=True)
source_id = db.Column(db.Integer,
db.ForeignKey('source.id'),
nullable=True)
dataset = db.relationship(Dataset,
backref=db.backref(
'runs',
order_by='Run.time_start.desc()',
lazy='dynamic'))
source = db.relationship(Source,
backref=db.backref(
'runs',
order_by='Run.time_start.desc()',
lazy='dynamic'))
def __init__(self, operation, status, dataset, source):
self.operation = operation
self.status = status
self.dataset = dataset
self.source = source
@property
def successful_sample(self):
"""
Returns True if the run was a sample operation (not full import)
and ran without failures.
"""
return self.operation == self.OPERATION_SAMPLE and \
self.status == self.STATUS_COMPLETE
@property
def successful_load(self):
"""
Returns True if the run was an import operation (not a sample)
and ran without failures.
"""
return self.operation == self.OPERATION_IMPORT and \
self.status == self.STATUS_COMPLETE
@property
def is_running(self):
"""
Returns True if the run is currently running
"""
return self.status == self.STATUS_RUNNING
@classmethod
def by_id(cls, id):
return db.session.query(cls).filter_by(id=id).first()
def __repr__(self):
return "<Run(%s,%s)>" % (self.source.id, self.id)
class Source(db.Model):
__tablename__ = 'source'
id = db.Column(db.Integer, primary_key=True)
url = db.Column(db.Unicode)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)
analysis = db.Column(MutableDict.as_mutable(JSONType), default=dict)
dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
dataset = db.relationship(Dataset,
backref=db.backref(
'sources',
lazy='dynamic',
order_by='Source.created_at.desc()'))
creator_id = db.Column(db.Integer, db.ForeignKey('account.id'))
creator = db.relationship(Account,
backref=db.backref('sources', lazy='dynamic'))
def __init__(self, dataset, creator, url):
self.dataset = dataset
self.creator = creator
self.url = url
@property
def loadable(self):
"""
Returns True if the source is ready to be imported into the
database. Does not not require a sample run although it
probably should.
"""
# It shouldn't be loaded again into the database
if self.successfully_loaded:
return False
# It needs mapping to be loadable
if not len(self.dataset.mapping):
return False
# There can be no errors in the analysis of the source
if 'error' in self.analysis:
return False
# All is good... proceed
return True
@property
def successfully_sampled(self):
"""
Returns True if any of this source's runs have been
successfully sampled (a complete sample run). This shows
whether the source is ready to be imported into the database
"""
return True in [r.successful_sample for r in self.runs]
@property
def is_running(self):
"""
Returns True if any of this source's runs have the status
'running'. This shows whether the loading has been started or not
to help avoid multiple loads of the same resource.
"""
return True in [r.is_running for r in self.runs]
@property
def successfully_loaded(self):
"""
Returns True if any of this source's runs have been
successfully loaded (not a sample and no errors). This
shows whether the source has been loaded into the database
"""
return True in [r.successful_load for r in self.runs]
def __repr__(self):
try:
return "<Source(%s,%s)>" % (self.dataset.name, self.url)
except:
return ''
@classmethod
def by_id(cls, id):
return db.session.query(cls).filter_by(id=id).first()
@classmethod
def all(cls):
return db.session.query(cls)
def as_dict(self):
return {
"id": self.id,
"url": self.url,
"dataset": self.dataset.name,
"created_at": self.created_at
}
