class Source(db.Model): __tablename__ = 'source' id = db.Column(db.Integer, primary_key=True) url = db.Column(db.Unicode) created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow) analysis = db.Column(JSONType, default=dict) dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id')) dataset = db.relationship(Dataset, backref=db.backref( 'sources', lazy='dynamic', order_by='Source.created_at.desc()')) creator_id = db.Column(db.Integer, db.ForeignKey('account.id')) creator = db.relationship(Account, backref=db.backref('sources', lazy='dynamic')) def __init__(self, dataset, creator, url): self.dataset = dataset self.creator = creator self.url = url @property def name(self): if len(self.url) > 55: return self.url[:15] + " .. " + self.url[len(self.url) - 25:] return self.url @property def loadable(self): if not len(self.dataset.mapping): return False if 'error' in self.analysis: return False return True def __repr__(self): return "<Source(%s,%s)>" % (self.dataset.name, self.name) @classmethod def by_id(cls, id): return db.session.query(cls).filter_by(id=id).first() def as_dict(self): return { "id": self.id, "url": self.url, "dataset": self.dataset.name, "created_at": self.created_at }
class LogRecord(db.Model): __tablename__ = 'log_record' CATEGORY_SYSTEM = 'system' CATEGORY_MODEL = 'model' CATEGORY_DATA = 'data' id = db.Column(db.Integer, primary_key=True) run_id = db.Column(db.Integer, db.ForeignKey('run.id')) category = db.Column(db.Unicode) level = db.Column(db.Unicode) message = db.Column(db.Unicode) error = db.Column(db.Unicode) timestamp = db.Column(db.DateTime, default=datetime.utcnow) row = db.Column(db.Integer) attribute = db.Column(db.Unicode) column = db.Column(db.Unicode) data_type = db.Column(db.Unicode) value = db.Column(db.Unicode) run = db.relationship(Run, backref=db.backref('records', lazy='dynamic')) def __init__(self, run, category, level, message): self.run = run self.category = category self.level = level self.message = message @classmethod def by_id(cls, id): return db.session.query(cls).filter_by(id=id).first()
class Run(db.Model): """ A run is a generic grouping object for background operations that perform logging to the frontend. """ __tablename__ = 'run' STATUS_RUNNING = 'running' STATUS_COMPLETE = 'complete' STATUS_FAILED = 'failed' id = db.Column(db.Integer, primary_key=True) operation = db.Column(db.Unicode(2000)) status = db.Column(db.Unicode(2000)) time_start = db.Column(db.DateTime, default=datetime.utcnow) time_end = db.Column(db.DateTime) dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'), nullable=True) source_id = db.Column(db.Integer, db.ForeignKey('source.id'), nullable=True) dataset = db.relationship(Dataset, backref=db.backref( 'runs', order_by='Run.time_start.desc()', lazy='dynamic')) source = db.relationship(Source, backref=db.backref( 'runs', order_by='Run.time_start.desc()', lazy='dynamic')) def __init__(self, operation, status, dataset, source): self.operation = operation self.status = status self.dataset = dataset self.source = source @classmethod def by_id(cls, id): return db.session.query(cls).filter_by(id=id).first() def __repr__(self): return "<Run(%s,%s)>" % (self.source.id, self.id)
class DatasetCountry(db.Model): __tablename__ = 'dataset_country' id = db.Column(db.Integer, primary_key=True) code = db.Column(db.Unicode) created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow) dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id')) dataset = db.relationship(Dataset, backref=db.backref('countries', lazy='dynamic')) def __init__(self, dataset, code): self.dataset = dataset self.code = code
class DatasetLanguage(db.Model, DatasetFacetMixin): __tablename__ = 'dataset_language' id = db.Column(db.Integer, primary_key=True) code = db.Column(db.Unicode) created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow) dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id')) dataset = db.relationship(Dataset, backref=db.backref('_languages', lazy=False)) def __init__(self, code): # self.dataset = dataset self.code = code
class Badge(db.Model): """ This model allows marking datasets with various badges. Examples could be "World Bank" - data verified by the World bank. Each badge has a name, a representative image and a description. Also stored for historical reasons are badge creator, creation time and modification date. """ __tablename__ = 'badge' id = db.Column(db.Integer, primary_key=True) # Primary information for this badge label = db.Column(db.Unicode) image = db.Column(db.Unicode) description = db.Column(db.Unicode) # Define relationship with datasets via the associate table datasets = db.relationship("Dataset", secondary=badges_on_datasets, backref=db.backref('badges', lazy='dynamic')) # Creator (and relationship) creator_id = db.Column(db.Integer, db.ForeignKey('account.id')) creator = db.relationship(Account, backref=db.backref('badge_creations', lazy='dynamic')) # Timestamps created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) def __init__(self, label, image, description, creator): """ Initialize a badge object. Badge label should be a representative title for the badge Image should be a small, representative image for the badge Description describes the purpose of the badge in more detail Creator is the user who created the badge. """ self.label = label self.image = image self.description = description self.creator = creator def __repr__(self): return "<Badge(%s)>" % self.label @classmethod def by_id(cls, id): """ Find one badge given the id """ return db.session.query(cls).filter_by(id=id).first() @classmethod def all(cls): """ Find all badges """ return db.session.query(cls) def as_dict(self, short=False): """ A dictionary representation of the badge. This can return a long version containing all interesting fields or a short version containing only id, label and image. """ badge = { "id": self.id, "label": self.label, "image": self.image, } if not short: badge.update({ "description": self.description, "datasets": [ds.name for ds in self.datasets], "created_at": self.created_at }) return badge
from datetime import datetime from openspending.model import Account, meta as db # Badges and dataset share a many to many relationship # therefore we need to create an associate table badges_on_datasets = db.Table( 'badges_on_datasets', db.metadata, db.Column('badge_id', db.Integer, db.ForeignKey('badge.id')), db.Column('dataset_id', db.Integer, db.ForeignKey('dataset.id'))) class Badge(db.Model): """ This model allows marking datasets with various badges. Examples could be "World Bank" - data verified by the World bank. Each badge has a name, a representative image and a description. Also stored for historical reasons are badge creator, creation time and modification date. """ __tablename__ = 'badge' id = db.Column(db.Integer, primary_key=True) # Primary information for this badge label = db.Column(db.Unicode) image = db.Column(db.Unicode) description = db.Column(db.Unicode) # Define relationship with datasets via the associate table
from openspending.model import meta as db from openspending.model.dataset import Dataset REGISTER_NAME_RE = r"^[a-zA-Z0-9_\-]{3,255}$" def make_uuid(): return unicode(uuid.uuid4()) account_dataset_table = db.Table( 'account_dataset', db.metadata, db.Column('dataset_id', db.Integer, db.ForeignKey('dataset.id'), primary_key=True), db.Column('account_id', db.Integer, db.ForeignKey('account.id'), primary_key=True)) class Account(db.Model): __tablename__ = 'account' id = db.Column(db.Integer, primary_key=True) name = db.Column(db.Unicode(255), unique=True) fullname = db.Column(db.Unicode(2000)) email = db.Column(db.Unicode(2000)) password = db.Column(db.Unicode(2000))
class View(db.Model): """ A view stores a specific configuration of a visualisation widget. """ __tablename__ = 'view' id = db.Column(db.Integer, primary_key=True) widget = db.Column(db.Unicode(2000)) name = db.Column(db.Unicode(2000)) label = db.Column(db.Unicode(2000)) description = db.Column(db.Unicode()) state = db.Column(MutableDict.as_mutable(JSONType), default=dict) public = db.Column(db.Boolean, default=False) created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow) dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id')) account_id = db.Column(db.Integer, db.ForeignKey('account.id'), nullable=True) dataset = db.relationship(Dataset, backref=db.backref( 'views', cascade='all,delete,delete-orphan', lazy='dynamic')) account = db.relationship(Account, backref=db.backref( 'views', cascade='all,delete,delete-orphan', lazy='dynamic')) def __init__(self): pass @classmethod def by_id(cls, id): return db.session.query(cls).filter_by(id=id).first() @classmethod def by_name(cls, dataset, name): q = db.session.query(cls).filter_by(name=name) return q.filter_by(dataset=dataset).first() @classmethod def all_by_dataset(cls, dataset): return db.session.query(cls).filter_by(dataset=dataset) def as_dict(self): return { 'id': self.id, 'widget': self.widget, 'name': self.name, 'label': self.label, 'description': self.description, 'state': self.state, 'public': self.public, 'dataset': self.dataset.name, 'account': self.account.name if self.account else None } def __repr__(self): return "<View(%s,%s)>" % (self.dataset.name, self.name)
class Run(db.Model): """ A run is a generic grouping object for background operations that perform logging to the frontend. """ __tablename__ = 'run' # Status values STATUS_RUNNING = 'running' STATUS_COMPLETE = 'complete' STATUS_FAILED = 'failed' STATUS_REMOVED = 'removed' # Operation values for database, two operations possible OPERATION_SAMPLE = 'sample' OPERATION_IMPORT = 'import' id = db.Column(db.Integer, primary_key=True) operation = db.Column(db.Unicode(2000)) status = db.Column(db.Unicode(2000)) time_start = db.Column(db.DateTime, default=datetime.utcnow) time_end = db.Column(db.DateTime) dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'), nullable=True) source_id = db.Column(db.Integer, db.ForeignKey('source.id'), nullable=True) dataset = db.relationship(Dataset, backref=db.backref( 'runs', order_by='Run.time_start.desc()', lazy='dynamic')) source = db.relationship(Source, backref=db.backref( 'runs', order_by='Run.time_start.desc()', lazy='dynamic')) def __init__(self, operation, status, dataset, source): self.operation = operation self.status = status self.dataset = dataset self.source = source @property def successful_sample(self): """ Returns True if the run was a sample operation (not full import) and ran without failures. """ return self.operation == self.OPERATION_SAMPLE and \ self.status == self.STATUS_COMPLETE @property def successful_load(self): """ Returns True if the run was an import operation (not a sample) and ran without failures. """ return self.operation == self.OPERATION_IMPORT and \ self.status == self.STATUS_COMPLETE @property def is_running(self): """ Returns True if the run is currently running """ return self.status == self.STATUS_RUNNING @classmethod def by_id(cls, id): return db.session.query(cls).filter_by(id=id).first() def __repr__(self): return "<Run(%s,%s)>" % (self.source.id, self.id)
class Source(db.Model): __tablename__ = 'source' id = db.Column(db.Integer, primary_key=True) url = db.Column(db.Unicode) created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow) analysis = db.Column(MutableDict.as_mutable(JSONType), default=dict) dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id')) dataset = db.relationship(Dataset, backref=db.backref( 'sources', lazy='dynamic', order_by='Source.created_at.desc()')) creator_id = db.Column(db.Integer, db.ForeignKey('account.id')) creator = db.relationship(Account, backref=db.backref('sources', lazy='dynamic')) def __init__(self, dataset, creator, url): self.dataset = dataset self.creator = creator self.url = url @property def loadable(self): """ Returns True if the source is ready to be imported into the database. Does not not require a sample run although it probably should. """ # It shouldn't be loaded again into the database if self.successfully_loaded: return False # It needs mapping to be loadable if not len(self.dataset.mapping): return False # There can be no errors in the analysis of the source if 'error' in self.analysis: return False # All is good... proceed return True @property def successfully_sampled(self): """ Returns True if any of this source's runs have been successfully sampled (a complete sample run). This shows whether the source is ready to be imported into the database """ return True in [r.successful_sample for r in self.runs] @property def is_running(self): """ Returns True if any of this source's runs have the status 'running'. This shows whether the loading has been started or not to help avoid multiple loads of the same resource. """ return True in [r.is_running for r in self.runs] @property def successfully_loaded(self): """ Returns True if any of this source's runs have been successfully loaded (not a sample and no errors). This shows whether the source has been loaded into the database """ return True in [r.successful_load for r in self.runs] def __repr__(self): try: return "<Source(%s,%s)>" % (self.dataset.name, self.url) except: return '' @classmethod def by_id(cls, id): return db.session.query(cls).filter_by(id=id).first() @classmethod def all(cls): return db.session.query(cls) def as_dict(self): return { "id": self.id, "url": self.url, "dataset": self.dataset.name, "created_at": self.created_at }