Exemplo n.º 1
0
class Source(db.Model):
    __tablename__ = 'source'

    id = db.Column(db.Integer, primary_key=True)
    url = db.Column(db.Unicode)
    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)
    analysis = db.Column(JSONType, default=dict)

    dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
    dataset = db.relationship(Dataset,
                              backref=db.backref(
                                  'sources',
                                  lazy='dynamic',
                                  order_by='Source.created_at.desc()'))

    creator_id = db.Column(db.Integer, db.ForeignKey('account.id'))
    creator = db.relationship(Account,
                              backref=db.backref('sources', lazy='dynamic'))

    def __init__(self, dataset, creator, url):
        self.dataset = dataset
        self.creator = creator
        self.url = url

    @property
    def name(self):
        if len(self.url) > 55:
            return self.url[:15] + " .. " + self.url[len(self.url) - 25:]
        return self.url

    @property
    def loadable(self):
        if not len(self.dataset.mapping):
            return False
        if 'error' in self.analysis:
            return False
        return True

    def __repr__(self):
        return "<Source(%s,%s)>" % (self.dataset.name, self.name)

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()

    def as_dict(self):
        return {
            "id": self.id,
            "url": self.url,
            "dataset": self.dataset.name,
            "created_at": self.created_at
        }
Exemplo n.º 2
0
class Account(db.Model):
    __tablename__ = 'account'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Unicode(255), unique=True)
    fullname = db.Column(db.Unicode(2000))
    email = db.Column(db.Unicode(2000))
    password = db.Column(db.Unicode(2000))
    api_key = db.Column(db.Unicode(2000), default=make_uuid)
    admin = db.Column(db.Boolean, default=False)

    datasets = db.relationship(Dataset,
                               secondary=account_dataset_table,
                               backref=db.backref('managers', lazy='dynamic'))

    def __init__(self):
        pass

    @classmethod
    def by_name(cls, name):
        return db.session.query(cls).filter_by(name=name).first()

    @classmethod
    def by_api_key(cls, api_key):
        return db.session.query(cls).filter_by(api_key=api_key).first()

    def as_dict(self):
        return {
            'name': self.name,
            'fullname': self.fullname,
            'email': self.email,
            'admin': self.admin
        }
Exemplo n.º 3
0
class LogRecord(db.Model):
    __tablename__ = 'log_record'

    CATEGORY_SYSTEM = 'system'
    CATEGORY_MODEL = 'model'
    CATEGORY_DATA = 'data'

    id = db.Column(db.Integer, primary_key=True)
    run_id = db.Column(db.Integer, db.ForeignKey('run.id'))

    category = db.Column(db.Unicode)
    level = db.Column(db.Unicode)
    message = db.Column(db.Unicode)
    error = db.Column(db.Unicode)
    timestamp = db.Column(db.DateTime, default=datetime.utcnow)

    row = db.Column(db.Integer)
    attribute = db.Column(db.Unicode)
    column = db.Column(db.Unicode)
    data_type = db.Column(db.Unicode)
    value = db.Column(db.Unicode)

    run = db.relationship(Run, backref=db.backref('records', lazy='dynamic'))

    def __init__(self, run, category, level, message):
        self.run = run
        self.category = category
        self.level = level
        self.message = message

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()
Exemplo n.º 4
0
class Run(db.Model):
    """ A run is a generic grouping object for background operations
    that perform logging to the frontend. """

    __tablename__ = 'run'

    STATUS_RUNNING = 'running'
    STATUS_COMPLETE = 'complete'
    STATUS_FAILED = 'failed'

    id = db.Column(db.Integer, primary_key=True)
    operation = db.Column(db.Unicode(2000))
    status = db.Column(db.Unicode(2000))
    time_start = db.Column(db.DateTime, default=datetime.utcnow)
    time_end = db.Column(db.DateTime)
    dataset_id = db.Column(db.Integer,
                           db.ForeignKey('dataset.id'),
                           nullable=True)
    source_id = db.Column(db.Integer,
                          db.ForeignKey('source.id'),
                          nullable=True)

    dataset = db.relationship(Dataset,
                              backref=db.backref(
                                  'runs',
                                  order_by='Run.time_start.desc()',
                                  lazy='dynamic'))
    source = db.relationship(Source,
                             backref=db.backref(
                                 'runs',
                                 order_by='Run.time_start.desc()',
                                 lazy='dynamic'))

    def __init__(self, operation, status, dataset, source):
        self.operation = operation
        self.status = status
        self.dataset = dataset
        self.source = source

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()

    def __repr__(self):
        return "<Run(%s,%s)>" % (self.source.id, self.id)
Exemplo n.º 5
0
class Account(db.Model):
    __tablename__ = 'account'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Unicode(255), unique=True)
    fullname = db.Column(db.Unicode(2000))
    email = db.Column(db.Unicode(2000))
    password = db.Column(db.Unicode(2000))
    api_key = db.Column(db.Unicode(2000), default=make_uuid)
    admin = db.Column(db.Boolean, default=False)
    script_root = db.Column(db.Unicode(2000))
    terms = db.Column(db.Boolean, default=False)

    datasets = db.relationship(Dataset,
                               secondary=account_dataset_table,
                               backref=db.backref('managers', lazy='dynamic'))

    def __init__(self):
        pass

    @property
    def display_name(self):
        return self.fullname or self.name

    @property
    def token(self):
        h = hmac.new('')
        h.update(self.api_key)
        if self.password:
            h.update(self.password)
        return h.hexdigest()

    @classmethod
    def by_name(cls, name):
        return db.session.query(cls).filter_by(name=name).first()

    @classmethod
    def by_email(cls, email):
        return db.session.query(cls).filter_by(email=email).first()

    @classmethod
    def by_api_key(cls, api_key):
        return db.session.query(cls).filter_by(api_key=api_key).first()

    def as_dict(self):
        return {
            'name': self.name,
            'fullname': self.fullname,
            'email': self.email,
            'admin': self.admin
        }
Exemplo n.º 6
0
class DatasetCountry(db.Model):
    __tablename__ = 'dataset_country'

    id = db.Column(db.Integer, primary_key=True)
    code = db.Column(db.Unicode)
    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)

    dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
    dataset = db.relationship(Dataset,
                              backref=db.backref('countries', lazy='dynamic'))

    def __init__(self, dataset, code):
        self.dataset = dataset
        self.code = code
Exemplo n.º 7
0
class DatasetLanguage(db.Model, DatasetFacetMixin):
    __tablename__ = 'dataset_language'

    id = db.Column(db.Integer, primary_key=True)
    code = db.Column(db.Unicode)
    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)

    dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
    dataset = db.relationship(Dataset,
                              backref=db.backref('_languages', lazy=False))

    def __init__(self, code):
        # self.dataset = dataset
        self.code = code
Exemplo n.º 8
0
class Badge(db.Model):
    """
    This model allows marking datasets with various badges.
    Examples could be "World Bank" - data verified by the World bank.

    Each badge has a name, a representative image and a description.
    Also stored for historical reasons are badge creator, creation time
    and modification date.
    """
    __tablename__ = 'badge'

    id = db.Column(db.Integer, primary_key=True)

    # Primary information for this badge
    label = db.Column(db.Unicode)
    image = db.Column(db.Unicode)
    description = db.Column(db.Unicode)

    # Define relationship with datasets via the associate table
    datasets = db.relationship("Dataset",
                               secondary=badges_on_datasets,
                               backref=db.backref('badges', lazy='dynamic'))

    # Creator (and relationship)
    creator_id = db.Column(db.Integer, db.ForeignKey('account.id'))
    creator = db.relationship(Account,
                              backref=db.backref('badge_creations',
                                                 lazy='dynamic'))

    # Timestamps
    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime,
                           default=datetime.utcnow,
                           onupdate=datetime.utcnow)

    def __init__(self, label, image, description, creator):
        """
        Initialize a badge object.
        Badge label should be a representative title for the badge
        Image should be a small, representative image for the badge
        Description describes the purpose of the badge in more detail
        Creator is the user who created the badge.
        """

        self.label = label
        self.image = image
        self.description = description
        self.creator = creator

    def __repr__(self):
        return "<Badge(%s)>" % self.label

    @classmethod
    def by_id(cls, id):
        """
        Find one badge given the id
        """
        return db.session.query(cls).filter_by(id=id).first()

    @classmethod
    def all(cls):
        """
        Find all badges
        """
        return db.session.query(cls)

    def as_dict(self, short=False):
        """
        A dictionary representation of the badge. This can return a long
        version containing all interesting fields or a short version containing
        only id, label and image.
        """
        badge = {
            "id": self.id,
            "label": self.label,
            "image": self.image,
        }
        if not short:
            badge.update({
                "description": self.description,
                "datasets": [ds.name for ds in self.datasets],
                "created_at": self.created_at
            })

        return badge
Exemplo n.º 9
0
class View(db.Model):
    """ A view stores a specific configuration of a visualisation widget. """

    __tablename__ = 'view'

    id = db.Column(db.Integer, primary_key=True)
    widget = db.Column(db.Unicode(2000))
    name = db.Column(db.Unicode(2000))
    label = db.Column(db.Unicode(2000))
    description = db.Column(db.Unicode())
    state = db.Column(MutableDict.as_mutable(JSONType), default=dict)
    public = db.Column(db.Boolean, default=False)

    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)

    dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
    account_id = db.Column(db.Integer,
                           db.ForeignKey('account.id'),
                           nullable=True)

    dataset = db.relationship(Dataset,
                              backref=db.backref(
                                  'views',
                                  cascade='all,delete,delete-orphan',
                                  lazy='dynamic'))

    account = db.relationship(Account,
                              backref=db.backref(
                                  'views',
                                  cascade='all,delete,delete-orphan',
                                  lazy='dynamic'))

    def __init__(self):
        pass

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()

    @classmethod
    def by_name(cls, dataset, name):
        q = db.session.query(cls).filter_by(name=name)
        return q.filter_by(dataset=dataset).first()

    @classmethod
    def all_by_dataset(cls, dataset):
        return db.session.query(cls).filter_by(dataset=dataset)

    def as_dict(self):
        return {
            'id': self.id,
            'widget': self.widget,
            'name': self.name,
            'label': self.label,
            'description': self.description,
            'state': self.state,
            'public': self.public,
            'dataset': self.dataset.name,
            'account': self.account.name if self.account else None
        }

    def __repr__(self):
        return "<View(%s,%s)>" % (self.dataset.name, self.name)
Exemplo n.º 10
0
class Run(db.Model):
    """ A run is a generic grouping object for background operations
    that perform logging to the frontend. """

    __tablename__ = 'run'

    # Status values
    STATUS_RUNNING = 'running'
    STATUS_COMPLETE = 'complete'
    STATUS_FAILED = 'failed'
    STATUS_REMOVED = 'removed'

    # Operation values for database, two operations possible
    OPERATION_SAMPLE = 'sample'
    OPERATION_IMPORT = 'import'

    id = db.Column(db.Integer, primary_key=True)
    operation = db.Column(db.Unicode(2000))
    status = db.Column(db.Unicode(2000))
    time_start = db.Column(db.DateTime, default=datetime.utcnow)
    time_end = db.Column(db.DateTime)
    dataset_id = db.Column(db.Integer,
                           db.ForeignKey('dataset.id'),
                           nullable=True)
    source_id = db.Column(db.Integer,
                          db.ForeignKey('source.id'),
                          nullable=True)

    dataset = db.relationship(Dataset,
                              backref=db.backref(
                                  'runs',
                                  order_by='Run.time_start.desc()',
                                  lazy='dynamic'))
    source = db.relationship(Source,
                             backref=db.backref(
                                 'runs',
                                 order_by='Run.time_start.desc()',
                                 lazy='dynamic'))

    def __init__(self, operation, status, dataset, source):
        self.operation = operation
        self.status = status
        self.dataset = dataset
        self.source = source

    @property
    def successful_sample(self):
        """
        Returns True if the run was a sample operation (not full import)
        and ran without failures.
        """
        return self.operation == self.OPERATION_SAMPLE and \
            self.status == self.STATUS_COMPLETE

    @property
    def successful_load(self):
        """
        Returns True if the run was an import operation (not a sample)
        and ran without failures.
        """
        return self.operation == self.OPERATION_IMPORT and \
            self.status == self.STATUS_COMPLETE

    @property
    def is_running(self):
        """
        Returns True if the run is currently running
        """
        return self.status == self.STATUS_RUNNING

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()

    def __repr__(self):
        return "<Run(%s,%s)>" % (self.source.id, self.id)
Exemplo n.º 11
0
class Account(db.Model):
    __tablename__ = 'account'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Unicode(255), unique=True)
    fullname = db.Column(db.Unicode(2000))
    email = db.Column(db.Unicode(2000))
    public_email = db.Column(db.Boolean, default=False)
    twitter_handle = db.Column(db.Unicode(140))
    public_twitter = db.Column(db.Boolean, default=False)
    password = db.Column(db.Unicode(2000))
    api_key = db.Column(db.Unicode(2000), default=make_uuid)
    admin = db.Column(db.Boolean, default=False)
    script_root = db.Column(db.Unicode(2000))
    terms = db.Column(db.Boolean, default=False)

    datasets = db.relationship(Dataset,
                               secondary=account_dataset_table,
                               backref=db.backref('managers', lazy='dynamic'))

    def __init__(self):
        pass

    @property
    def display_name(self):
        return self.fullname or self.name

    @property
    def token(self):
        h = hmac.new('')
        h.update(self.api_key)
        if self.password:
            h.update(self.password)
        return h.hexdigest()

    @classmethod
    def by_name(cls, name):
        return db.session.query(cls).filter_by(name=name).first()

    @classmethod
    def by_email(cls, email):
        return db.session.query(cls).filter_by(email=email).first()

    @classmethod
    def by_api_key(cls, api_key):
        return db.session.query(cls).filter_by(api_key=api_key).first()

    def as_dict(self):
        """
        Return the dictionary representation of the account
        """

        # Dictionary will include name, fullname, email and the admin bit
        account_dict = {
            'name': self.name,
            'fullname': self.fullname,
            'email': self.email,
            'admin': self.admin
        }

        # If the user has a twitter handle we add it
        if self.twitter_handle is not None:
            account_dict['twitter'] = self.twitter_handle

        # Return the dictionary representation
        return account_dict
Exemplo n.º 12
0
class Source(db.Model):
    __tablename__ = 'source'

    id = db.Column(db.Integer, primary_key=True)
    url = db.Column(db.Unicode)
    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime, onupdate=datetime.utcnow)
    analysis = db.Column(MutableDict.as_mutable(JSONType), default=dict)

    dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'))
    dataset = db.relationship(Dataset,
                              backref=db.backref(
                                  'sources',
                                  lazy='dynamic',
                                  order_by='Source.created_at.desc()'))

    creator_id = db.Column(db.Integer, db.ForeignKey('account.id'))
    creator = db.relationship(Account,
                              backref=db.backref('sources', lazy='dynamic'))

    def __init__(self, dataset, creator, url):
        self.dataset = dataset
        self.creator = creator
        self.url = url

    @property
    def loadable(self):
        """
        Returns True if the source is ready to be imported into the
        database. Does not not require a sample run although it
        probably should.
        """
        # It shouldn't be loaded again into the database
        if self.successfully_loaded:
            return False
        # It needs mapping to be loadable
        if not len(self.dataset.mapping):
            return False
        # There can be no errors in the analysis of the source
        if 'error' in self.analysis:
            return False
        # All is good... proceed
        return True

    @property
    def successfully_sampled(self):
        """
        Returns True if any of this source's runs have been
        successfully sampled (a complete sample run). This shows
        whether the source is ready to be imported into the database
        """
        return True in [r.successful_sample for r in self.runs]

    @property
    def is_running(self):
        """
        Returns True if any of this source's runs have the status
        'running'. This shows whether the loading has been started or not
        to help avoid multiple loads of the same resource.
        """
        return True in [r.is_running for r in self.runs]

    @property
    def successfully_loaded(self):
        """
        Returns True if any of this source's runs have been
        successfully loaded (not a sample and no errors). This
        shows whether the source has been loaded into the database
        """
        return True in [r.successful_load for r in self.runs]

    def __repr__(self):
        try:
            return "<Source(%s,%s)>" % (self.dataset.name, self.url)
        except:
            return ''

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()

    @classmethod
    def all(cls):
        return db.session.query(cls)

    def as_dict(self):
        return {
            "id": self.id,
            "url": self.url,
            "dataset": self.dataset.name,
            "created_at": self.created_at
        }