class Frame(db.Model):
__tablename__ = 'frame'
urn = db.Column(db.Unicode(), primary_key=True)
hash = db.Column(db.Unicode(), index=True)
service_id = db.Column(db.Integer(), db.ForeignKey('service.id'))
event_id = db.Column(db.Integer(), db.ForeignKey('event.id'))
action_at = db.Column(db.DateTime, index=True)
submitted_at = db.Column(db.DateTime, index=True)
matches = db.relationship('Match', backref='frame', lazy='dynamic',
cascade='all, delete-orphan', order_by='Match.created_at.desc()')
@classmethod
def create(cls, service, event, data):
obj = cls()
obj.urn = data.get('urn')
obj.hash = data.get('hash')
obj.submitted_at = data.get('submitted_at')
obj.action_at = data.get('action_at')
obj.service = service
obj.event = event
db.session.add(obj)
return obj
@classmethod
def to_urn(cls, frame):
uuid = make_token()
instance = app.config.get('INSTANCE', 'dwre')
return 'urn:%s:%s:%s:%s' % (instance, frame['service'],
frame['event'], uuid)
@classmethod
def by_hash(cls, hash):
q = db.session.query(cls).filter_by(hash=hash)
return q.first()
@classmethod
def by_urn(cls, urn):
q = db.session.query(cls).filter_by(urn=urn)
return q.first()
def to_ref(self):
from datawire.store import frame_url
return {
'urn': self.urn,
'api_uri': url_for('frames.get', urn=self.urn, _external=True),
'store_uri': frame_url(self.urn),
'action_at': self.action_at,
'submitted_at': self.submitted_at
}
@classmethod
def all(cls):
return db.session.query(cls)
class Match(db.Model, ModelCore):
__tablename__ = 'match'
urn = db.Column(db.Unicode(), db.ForeignKey('frame.urn'))
field = db.Column(db.Unicode())
entity_id = db.Column(db.Integer(), db.ForeignKey('entity.id'))
@classmethod
def create(cls, urn, field, entity_id):
obj = cls()
obj.urn = urn
obj.field = field
obj.entity_id = entity_id
db.session.add(obj)
return obj
@classmethod
def exists(cls, urn, entity_id):
q = db.session.query(cls)
q = q.filter_by(urn=urn)
q = q.filter_by(entity_id=entity_id)
return q.first()
def to_ref(self):
data = self.frame.to_ref()
data.update({
'id': self.id,
'field': self.field,
'matched_at': self.created_at,
'entity_id': self.entity_id
})
return data
def to_dict(self):
data = self.to_ref()
data.update({
'entity': self.entity
})
return data
class Collection(db.Model):
id = db.Column(db.Unicode(50), primary_key=True)
slug = db.Column(db.Unicode(250))
public = db.Column(db.Boolean, default=False)
owner_id = db.Column(db.Integer(), db.ForeignKey('user.id'), nullable=True)
owner = db.relationship(User)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime,
default=datetime.utcnow,
onupdate=datetime.utcnow)
def to_dict(self):
return {
'id': self.id,
'api_url': url_for('collections.view', id=self.id),
'entities_api_url': url_for('entities.index', list=self.id),
'slug': self.slug,
'public': self.public,
'owner': self.owner,
'created_at': self.created_at,
'updated_at': self.updated_at
}
@classmethod
def create(cls, data, user):
lst = cls()
lst.update(data, user)
lst.owner = user
db.session.add(lst)
return lst
def update(self, data, user):
data = CollectionForm().deserialize(data)
self.slug = data.get('slug')
if data.get('public') is not None:
self.public = data.get('public')
def delete(self):
# for entity in self.entities:
# entity.delete()
db.session.delete(self)
@classmethod
def by_slug(cls, login, slug):
q = db.session.query(cls).filter_by(slug=slug)
q = q.filter(cls.owner.login == login)
return q.first()
@classmethod
def by_id(cls, id):
q = db.session.query(cls).filter_by(id=id)
return q.first()
@classmethod
def user_ids(cls, user, include_public=True):
logged_in = user is not None and user.is_authenticated()
q = db.session.query(cls.id)
conds = []
if include_public:
conds.append(cls.public == True) # noqa
if logged_in:
conds.append(cls.owner_id == user.id)
if not len(conds):
return []
if not (logged_in and user.is_admin):
q = q.filter(or_(*conds))
return [c.id for c in q.all()]
@classmethod
def all_by_user(cls, user):
q = db.session.query(cls)
q = q.filter(cls.id.in_(cls.user_ids(user)))
q = q.order_by(cls.id.desc())
return q
@property
def terms(self):
from aleph.model.entity import Entity
from aleph.model.selector import Selector
q = db.session.query(Selector.normalized)
q = q.join(Entity, Entity.id == Selector.entity_id)
q = q.filter(Entity.watchlist_id == self.id)
q = q.distinct()
return set([r[0] for r in q])
def __repr__(self):
return '<Watchlist(%r, %r)>' % (self.id, self.slug)
def __unicode__(self):
return self.slug
class Entity(db.Model):
id = db.Column(db.Unicode(50), primary_key=True, default=make_textid)
label = db.Column(db.Unicode)
category = db.Column(db.Enum(*CATEGORIES, name='entity_categories'),
nullable=False)
creator_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
creator = db.relationship(User,
backref=db.backref('entities',
lazy='dynamic',
cascade='all, delete-orphan'))
collection_id = db.Column(db.Unicode(50), db.ForeignKey('collection.id'))
collection = db.relationship('Collection',
backref=db.backref(
'entities',
lazy='dynamic',
cascade='all, delete-orphan'))
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime,
default=datetime.utcnow,
onupdate=datetime.utcnow)
def to_dict(self):
return {
'id': self.id,
'api_url': url_for('entities.view', id=self.id),
'label': self.label,
'category': self.category,
'creator_id': self.creator_id,
'selectors': [s.text for s in self.selectors],
'collection_id': self.collection_id,
'created_at': self.created_at,
'updated_at': self.updated_at
}
def has_selector(self, text):
normalized = Selector.normalize(text)
for selector in self.selectors:
if selector.normalized == normalized:
return True
return False
def delete(self):
db.session.delete(self)
@classmethod
def create(cls, data, user):
ent = cls()
ent.update(data)
ent.creator = user
db.session.add(ent)
return ent
def update(self, data):
data = EntityForm().deserialize(data)
self.label = data.get('label')
self.collection = data.get('collection')
self.category = data.get('category')
selectors = set(data.get('selectors'))
selectors.add(self.label)
existing = list(self.selectors)
for sel in list(existing):
if sel.text in selectors:
selectors.remove(sel.text)
existing.remove(sel)
for sel in existing:
db.session.delete(sel)
for text in selectors:
sel = Selector()
sel.entity = self
sel.text = text
db.session.add(sel)
@classmethod
def by_normalized_label(cls, label, collection):
q = db.session.query(cls)
q = q.filter_by(collection=collection)
q = q.filter(db_compare(cls.label, label))
return q.first()
@classmethod
def by_id(cls, id):
q = db.session.query(cls).filter_by(id=id)
return q.first()
@classmethod
def by_collections(cls, collections, prefix=None):
q = db.session.query(cls)
q = q.filter(cls.collection_id.in_(collections))
if prefix is not None and len(prefix):
q = q.join(Selector, cls.id == Selector.entity_id)
q = cls.apply_filter(q, Selector.normalized, prefix)
q = q.order_by(cls.label.asc())
return q
@classmethod
def by_id_set(cls, ids):
if not len(ids):
return {}
q = db.session.query(cls)
q = q.filter(cls.id.in_(ids))
entities = {}
for ent in q:
entities[ent.id] = ent
return entities
@classmethod
def apply_filter(cls, q, col, prefix):
prefix = Selector.normalize(prefix)
return q.filter(
or_(col.like('%s%%' % prefix), col.like('%% %s%%' % prefix)))
@classmethod
def suggest_prefix(cls, prefix, collections, limit=10):
ent = aliased(Entity)
sel = aliased(Selector)
q = db.session.query(ent.id, ent.label, ent.category)
q = q.join(sel, ent.id == sel.entity_id)
q = q.filter(ent.collection_id.in_(collections))
if prefix is None or not len(prefix):
return []
q = cls.apply_filter(q, sel.normalized, prefix)
q = q.order_by(ent.label.asc())
q = q.limit(limit)
q = q.distinct()
suggestions = []
for entity_id, label, category in q.all():
suggestions.append({
'id': entity_id,
'label': label,
'category': category
})
return suggestions
@property
def terms(self):
return set([s.normalized for s in self.selectors])
def __repr__(self):
return '<Entity(%r, %r)>' % (self.id, self.label)
def __unicode__(self):
return self.label
class User(db.Model, ModelCore):
__tablename__ = 'user'
screen_name = db.Column(db.Unicode())
name = db.Column(db.Unicode())
email = db.Column(db.Unicode())
twitter_id = db.Column(db.Unicode())
facebook_id = db.Column(db.Unicode())
api_key = db.Column(db.Unicode(), default=make_token)
entities = db.relationship('Entity',
backref='user',
lazy='dynamic',
cascade='all, delete-orphan')
@classmethod
def create(cls, data):
obj = cls()
obj.screen_name = data.get('screen_name')
obj.name = data.get('name')
obj.twitter_id = data.get('twitter_id')
obj.facebook_id = data.get('facebook_id')
db.session.add(obj)
return obj
def update(self, data):
data = UserSchema().to_python(data)
self.name = data['name']
self.email = data['email']
db.session.add(self)
@classmethod
def by_screen_name(cls, screen_name):
q = db.session.query(cls)
q = q.filter_by(screen_name=screen_name)
return q.first()
@classmethod
def by_api_key(cls, api_key):
q = db.session.query(cls).filter_by(api_key=api_key)
return q.first()
@classmethod
def by_twitter_id(cls, twitter_id):
q = db.session.query(cls).filter_by(twitter_id=twitter_id)
return q.first()
@classmethod
def by_facebook_id(cls, facebook_id):
q = db.session.query(cls).filter_by(facebook_id=facebook_id)
return q.first()
def to_ref(self):
return {
'id': self.id,
'screen_name': self.screen_name,
'uri': url_for('users.get', id=self.id, _external=True),
'name': self.name
}
def to_dict(self):
# TODO: Do we want to expose entities?
data = self.to_ref()
data.update({
'created_at': self.created_at,
'updated_at': self.updated_at
})
return data
class Event(db.Model, ModelCore):
__tablename__ = 'event'
key = db.Column(db.Unicode())
label = db.Column(db.Unicode())
description = db.Column(db.Unicode())
template = db.Column(db.Unicode())
service_id = db.Column(db.Integer(), db.ForeignKey('service.id'))
frames = db.relationship('Frame',
backref='event',
lazy='dynamic',
cascade='all, delete-orphan',
order_by='Frame.action_at.desc()')
@classmethod
def create(cls, data):
# TODO: ensure this is the only event on this service with
# this key.
obj = cls()
obj.key = data.get('key')
obj.label = data.get('label')
obj.description = data.get('description')
obj.template = data.get('template')
obj.service = data.get('service')
db.session.add(obj)
return obj
def update(self, data):
self.label = data.get('label')
self.description = data.get('description')
self.template = data.get('template')
db.session.add(self)
@classmethod
def by_key(cls, service, key):
q = db.session.query(cls)
q = q.filter_by(service=service).filter_by(key=key)
return q.first()
def to_ref(self):
return {
'id':
self.id,
'key':
self.key,
'uri':
url_for('events.get',
service_key=self.service.key,
event_key=self.key,
_external=True),
'template':
self.template,
'label':
self.label
}
def to_dict(self):
data = self.to_ref()
data.update({
'description': self.description,
'created_at': self.created_at,
'updated_at': self.updated_at
})
return data
class Entity(db.Model, ModelCore):
__tablename__ = 'entity'
text = db.Column(db.Unicode())
category = db.Column(db.Unicode())
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
matches = db.relationship('Match', backref='entity', lazy='dynamic',
cascade='all, delete-orphan', order_by='Match.created_at.desc()')
@classmethod
def create(cls, data, user):
obj = cls()
data = EntitySchema().to_python(data)
obj.user = user
obj.text = data.get('text')
obj.category = data.get('category')
db.session.add(obj)
return obj
def update(self, data):
data = EntitySchema().to_python(data)
self.text = data.get('text')
self.category = data.get('category')
db.session.add(self)
def delete(self):
db.session.delete(self)
@property
def pattern(self):
return re.compile(self.text, re.I | re.M)
@classmethod
def by_user_and_id(cls, user, id):
q = db.session.query(cls).filter_by(user=user)
q = q.filter_by(id=id)
return q.first()
@classmethod
def by_text(cls, text):
q = db.session.query(cls).filter_by(text=text)
return q.first()
@classmethod
def by_category(cls, category):
q = db.session.query(cls).filter_by(category=category)
return q.first()
def to_ref(self):
return {
'id': self.id,
'text': self.text,
'category': self.category,
'user_id': self.user_id
}
def to_dict(self):
data = self.to_ref()
data.update({
'user': self.user.to_ref(),
'created_at': self.created_at,
'updated_at': self.updated_at
})
return data
class Service(db.Model, ModelCore):
__tablename__ = 'service'
key = db.Column(db.Unicode())
label = db.Column(db.Unicode())
description = db.Column(db.Unicode())
url = db.Column(db.Unicode())
frames = db.relationship('Frame',
backref='service',
lazy='dynamic',
cascade='all, delete-orphan',
order_by='Frame.action_at.desc()')
events = db.relationship('Event',
backref='service',
lazy='dynamic',
cascade='all, delete-orphan',
order_by='Event.key.asc()')
editors = db.relationship('User',
secondary=editors,
backref=db.backref('services', lazy='dynamic'))
@classmethod
def create(cls, data):
# TODO: ensure this is the only service with this key.
obj = cls()
obj.key = data.get('key')
obj.label = data.get('label')
obj.description = data.get('description')
obj.url = data.get('url')
db.session.add(obj)
return obj
def update(self, data):
self.label = data.get('label')
self.description = data.get('description')
self.url = data.get('url')
db.session.add(self)
@classmethod
def by_key(cls, key):
q = db.session.query(cls).filter_by(key=key)
return q.first()
def to_ref(self):
return {
'id': self.id,
'key': self.key,
'uri': url_for('services.get', key=self.key, _external=True),
'label': self.label
}
def to_dict(self):
data = self.to_ref()
data.update({
'created_at': self.created_at,
'updated_at': self.updated_at,
'description': self.description,
'url': self.url,
'events': [e.to_ref() for e in self.events],
'editors': self.editors
})
return data
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
login = db.Column(db.Unicode(255))
email = db.Column(db.Unicode, nullable=True)
oauth_id = db.Column(db.Unicode)
api_key = db.Column(db.Unicode, default=make_token)
is_admin = db.Column(db.Boolean, nullable=False, default=False)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, default=datetime.utcnow,
onupdate=datetime.utcnow)
def is_active(self):
return True
def is_authenticated(self):
return True
def is_anonymous(self):
return False
def get_id(self):
return self.login
def __repr__(self):
return '<User(%r,%r)>' % (self.id, self.login)
def __unicode__(self):
return self.login
def to_dict(self):
return {
'id': self.id,
'login': self.login,
'is_admin': self.is_admin,
'created_at': self.created_at,
'updated_at': self.updated_at,
'api_url': url_for('users.view', login=self.login)
}
def update(self, data):
data = UserForm().deserialize(data)
self.email = data.get('email')
@classmethod
def load(cls, data):
user = cls.by_oauth_id(data.get('oauth_id'))
if user is None:
user = cls()
user.login = data.get('login')
user.oauth_id = data.get('oauth_id')
if data.get('email'):
# FIXME: better to overwrite with upstream or keep?
user.email = data.get('email')
db.session.add(user)
return user
@classmethod
def all(cls):
q = db.session.query(cls)
return q
@classmethod
def by_login(cls, login):
q = db.session.query(cls).filter_by(login=login)
return q.first()
@classmethod
def by_api_key(cls, api_key):
q = db.session.query(cls).filter_by(api_key=api_key)
return q.first()
@classmethod
def by_oauth_id(cls, oauth_id):
q = db.session.query(cls).filter_by(oauth_id=unicode(oauth_id))
return q.first()