class Template(db.Model):
__tablename__ = 'templates'
__module__ = 'newslynx.models.template'
id = db.Column(db.Integer, unique=True, index=True, primary_key=True)
org_id = db.Column(db.Integer, db.ForeignKey('orgs.id'), index=True)
name = db.Column(db.Text)
slug = db.Column(db.Text, index=True)
created = db.Column(db.DateTime(timezone=True), default=dates.now)
updated = db.Column(db.DateTime(timezone=True),
onupdate=dates.now,
default=dates.now)
template = db.Column(db.Text)
format = db.Column(ENUM(*TEMPLATE_FORMATS, name="template_format_enum"))
reports = db.relationship('Report',
backref=db.backref('template', lazy='joined'),
lazy='dynamic',
cascade="all, delete-orphan")
__table_args__ = (db.UniqueConstraint('org_id', 'slug'), )
def __init__(self, **kw):
self.org_id = kw.get('org_id')
self.name = kw.get('name')
self.slug = kw.get('slug', slug(kw.get('name')))
self.template = kw.get('template')
self.format = kw.get('format')
self.data = kw.get('data')
def to_dict(self):
return {
'id': self.id,
'org_id': self.org_id,
'name': self.name,
'slug': self.slug,
'created': self.created,
'updated': self.updated,
'template': self.template,
'format': self.format
}
def render(self, **kw):
"""
Render this template.
"""
t = Tmpl(self.template)
return t.render(**kw)
def __repr__(self):
return "<Template %r / %r >" % (self.org_id, self.slug)
class Recipe(db.Model):
__tablename__ = 'recipes'
__module__ = 'newslynx.models.recipe'
# id fields
id = db.Column(db.Integer, unique=True, index=True, primary_key=True)
sous_chef_id = db.Column(
db.Integer, db.ForeignKey('sous_chefs.id'), index=True)
user_id = db.Column(
db.Integer, db.ForeignKey('users.id'), index=True)
org_id = db.Column(
db.Integer, db.ForeignKey('orgs.id'), index=True)
# core fields
name = db.Column(db.Text, index=True)
slug = db.Column(db.Text, index=True)
description = db.Column(db.Text)
# date fields
created = db.Column(db.DateTime(timezone=True), default=dates.now)
updated = db.Column(db.DateTime(timezone=True), default=dates.now, onupdate=dates.now)
last_run = db.Column(db.DateTime(timezone=True), index=True)
# scheduler fields
schedule_by = db.Column(ENUM(*RECIPE_SCHEDULE_TYPES, name="recipe_schedule_type_enum"), index=True)
crontab = db.Column(db.Text)
time_of_day = db.Column(db.Text)
minutes = db.Column(db.Integer)
status = db.Column(
ENUM(*RECIPE_STATUSES, name="enum_recipe_statuses"), index=True)
traceback = db.Column(db.Text)
last_job = db.Column(JSON)
# options
options = db.Column(db.Text)
options_hash = db.Column(db.Text)
# relations
events = db.relationship('Event', lazy='dynamic')
content_items = db.relationship('ContentItem', lazy='dynamic')
metrics = db.relationship('Metric', backref=db.backref('recipe', lazy='joined'), lazy='joined')
sous_chef = db.relationship(
'SousChef', backref=db.backref('recipes', lazy='joined', cascade="all, delete-orphan"), lazy='joined')
user = db.relationship(
'User', backref=db.backref('recipes', lazy='dynamic'), lazy='joined')
__table_args__ = (
db.UniqueConstraint('org_id', 'name'),
)
def __init__(self, sous_chef, **kw):
"""
A recipe must be initialized with an existing sous chef.
"""
# core fields
self.name = kw.get('name')
self.slug = slug(kw.get('slug', kw['name']))
self.description = kw.get('description')
self.schedule_by = kw.get('schedule_by', 'unscheduled')
self.crontab = kw.get('crontab')
self.time_of_day = kw.get('time_of_day')
self.minutes = kw.get('minutes')
self.status = kw.get('status', 'stable')
self.traceback = kw.get('traceback')
self.set_options(kw.get('options', {}))
# internal fields
self.sous_chef_id = sous_chef.id
self.user_id = kw.get('user_id')
self.org_id = kw.get('org_id')
self.last_run = kw.get('last_run', None)
self.last_job = kw.get('last_job', {})
def set_options(self, opts):
"""
pickle dump the options.
"""
p = obj_to_pickle(opts)
self.options = p
self.options_hash = str(md5(p).hexdigest())
@property
def scheduled(self):
"""
Is this recipe scheduled?
"""
return self.schedule_by != 'unscheduled'
@property
def active(self):
"""
Is this recipe scheduled?
"""
return self.status != 'inactive'
@property
def metric_names(self):
return [m.name for m in self.metrics]
@property
def report_names(self):
return [r.slug for r in self.reports]
def to_dict(self, **kw):
incl_reports = kw.get("incl_reports", True)
d = {
'id': self.id,
'org_id': self.org_id,
'sous_chef': self.sous_chef.slug,
'name': self.name,
'slug': self.slug,
'description': self.description,
'created': self.created,
'updated': self.updated,
'last_run': self.last_run,
'schedule_by': self.schedule_by,
'crontab': self.crontab,
'time_of_day': self.time_of_day,
'minutes': self.minutes,
'status': self.status,
'traceback': self.traceback,
'last_job': self.last_job,
'options': pickle_to_obj(self.options)
}
if 'metrics' in self.sous_chef.creates:
d['metrics'] = self.metric_names
if incl_reports:
if 'report' in self.sous_chef.creates:
d['reports'] = self.report_names
return d
def __repr__(self):
return '<Recipe %r >' % (self.slug)
class ContentItem(db.Model):
"""
A content-item is a unit of content
to which we attach metrics.
We do not initialize a content-item until we have past it completely through
our single ingestion pipeline.
At this point all content-items should have a standardized schema,
though may not have all theses fields filled in.
"""
query_class = SearchQuery
__tablename__ = 'content'
__module__ = 'newslynx.models.content_item'
# the ID is the global bitly hash.
id = db.Column(db.Integer, unique=True, primary_key=True, index=True)
org_id = db.Column(
db.Integer, db.ForeignKey('orgs.id'), index=True)
recipe_id = db.Column(db.Integer, db.ForeignKey('recipes.id'), index=True)
type = db.Column(ENUM(*CONTENT_ITEM_TYPES, name='content_item_types_enum'))
provenance = db.Column(
ENUM(*CONTENT_ITEM_PROVENANCES, name='content_item_provenance_enum'), index=True)
url = db.Column(db.Text, index=True)
domain = db.Column(db.Text, index=True)
created = db.Column(db.DateTime(timezone=True), default=dates.now)
updated = db.Column(
db.DateTime(timezone=True), onupdate=dates.now, default=dates.now)
site_name = db.Column(db.Text, index=True)
favicon = db.Column(db.Text)
img_url = db.Column(db.Text)
thumbnail = db.Column(db.Text)
title = db.Column(db.Text)
description = db.Column(db.Text)
body = db.Column(db.Text)
active = db.Column(db.Boolean, index=True)
meta = db.Column(JSON)
# relations
tags = db.relationship(
'Tag', secondary=relations.content_items_tags,
backref=db.backref('content_items', lazy='dynamic'), lazy='joined')
events = db.relationship(
'Event',
secondary=relations.content_items_events,
backref=db.backref('content_items', lazy='dynamic'),
lazy='dynamic')
authors = db.relationship(
'Author', secondary=relations.content_items_authors,
backref=db.backref('content_items', lazy='dynamic'), lazy='joined')
summary_metrics = db.relationship(
'ContentMetricSummary', lazy='joined', uselist=False, cascade="all, delete-orphan")
timeseries_metrics = db.relationship(
'ContentMetricTimeseries', lazy='dynamic', cascade="all, delete-orphan")
# # in/out links
# out_links = db.relationship(
# 'ContentItem', secondary=relations.content_items_content_items,
# primaryjoin=relations.content_items_content_items.c.from_content_item_id == id,
# secondaryjoin=relations.content_items_content_items.c.to_content_item_id == id,
# backref=db.backref("in_links", lazy='dynamic'),
# lazy='dynamic')
# search vectors
title_search_vector = db.Column(TSVectorType('title'))
body_search_vector = db.Column(TSVectorType('body'))
description_search_vector = db.Column(TSVectorType('description'))
meta_search_vector = db.Column(TSVectorType('meta'))
# content_items should be unique to org, url, and type.
# IE there might be multiple content_items per url -
# an article, a video, a podcast, etc.
__table_args__ = (
db.UniqueConstraint(
'org_id', 'url', 'type', name='content_item_unique_constraint'),
Index('content_item_title_search_vector_idx',
'title_search_vector', postgresql_using='gin'),
Index('content_item_body_search_vector_idx',
'body_search_vector', postgresql_using='gin'),
Index('content_item_description_search_vector_idx',
'description_search_vector', postgresql_using='gin'),
Index('content_item_meta_search_vector_idx',
'meta_search_vector', postgresql_using='gin')
)
def __init__(self, **kw):
self.org_id = kw.get('org_id')
self.recipe_id = kw.get('recipe_id')
self.url = kw.get('url')
self.type = kw.get('type')
self.provenance = kw.get('provenance', 'recipe')
self.domain = kw.get('domain')
self.created = kw.get('created', dates.now())
self.site_name = kw.get('site_name')
self.favicon = kw.get('favicon')
self.img_url = kw.get('img_url')
self.thumbnail = kw.get('thumbnail')
self.title = kw.get('title')
self.description = kw.get('description')
self.body = kw.get('body')
self.active = kw.get('active', True)
self.meta = kw.get('meta', {})
@property
def simple_authors(self):
return [{"id": a.id, "name": a.name} for a in self.authors]
@property
def author_ids(self):
return [a.id for a in self.authors]
# @property
# def out_link_ids(self):
# out_links = db.session.query(relations.content_items_content_items.c.to_content_item_id)\
# .filter(relations.content_items_content_items.c.from_content_item_id == self.id)\
# .all()
# return [o[0] for o in out_links]
# @property
# def in_link_ids(self):
# in_links = db.session.query(relations.content_items_content_items.c.from_content_item_id)\
# .filter(relations.content_items_content_items.c.to_content_item_id == self.id)\
# .all()
# return [o[0] for o in in_links]
# @property
# def out_link_display(self):
# out_links = self.out_links\
# .with_entities(ContentItem.id, ContentItem.title)\
# .all()
# return [dict(zip(['id', 'title'], l)) for l in out_links]
# @property
# def in_link_display(self):
# in_links = self.in_links\
# .with_entities(ContentItem.id, ContentItem.title)\
# .all()
# return [dict(zip(['id', 'title'], l)) for l in in_links]
@property
def tag_ids(self):
return [t.id for t in self.tags]
@property
def subject_tag_ids(self):
return [t.id for t in self.tags if t.type == 'subject']
@property
def impact_tag_ids(self):
return [t.id for e in self.events for t in e.tags if t.type == 'impact']
@property
def event_ids(self):
return [e.id for e in self.events]
def to_dict(self, **kw):
# incl_links = kw.get('incl_links', False)
incl_body = kw.get('incl_body', False)
incl_metrics = kw.get('incl_metrics', True)
incl_img = kw.get('incl_img', False)
d = {
'id': self.id,
'org_id': self.org_id,
'recipe_id': self.recipe_id,
'url': self.url,
'domain': self.domain,
'provenance': self.provenance,
'type': self.type,
'created': self.created,
'updated': self.updated,
'favicon': self.favicon,
'site_name': self.site_name,
'authors': self.simple_authors,
'title': self.title,
'description': self.description,
'subject_tag_ids': self.subject_tag_ids,
'impact_tag_ids': self.impact_tag_ids,
'active': self.active,
'meta': self.meta
}
# if incl_links:
# d['in_links'] = self.in_link_display
# d['out_links'] = self.out_link_display
if incl_body:
d['body'] = self.body
if incl_metrics:
if self.summary_metrics:
d['metrics'] = self.summary_metrics.metrics
else:
d['metrics'] = {}
if incl_img:
d['thumbnail'] = self.thumbnail
d['img_url'] = self.img_url
return d
def __repr__(self):
return '<ContentItem %r / %r >' % (self.url, self.type)
class Org(db.Model):
__tablename__ = 'orgs'
__module__ = 'newslynx.models.org'
id = db.Column(db.Integer, unique=True, index=True, primary_key=True)
name = db.Column(db.Text, unique=True, index=True)
slug = db.Column(db.Text, unique=True, index=True)
timezone = db.Column(
ENUM(*list(dates.TIMEZONES), name='org_timezones_enum'))
created = db.Column(db.DateTime(timezone=True), default=dates.now)
updated = db.Column(db.DateTime(timezone=True),
onupdate=dates.now,
default=dates.now)
# joins
auths = db.relationship('Auth',
backref=db.backref('org'),
lazy='joined',
cascade="all, delete-orphan")
settings = db.relationship('Setting',
backref=db.backref('org'),
lazy='joined',
cascade="all, delete-orphan")
# dynamic relations
reports = db.relationship('Report',
backref=db.backref('org'),
lazy='dynamic',
cascade="all, delete-orphan")
users = db.relationship('User',
secondary=orgs_users,
backref=db.backref('orgs', lazy='joined'),
lazy='joined')
events = db.relationship('Event', lazy='dynamic', cascade='all')
content_items = db.relationship('ContentItem',
lazy='dynamic',
cascade='all')
sous_chefs = db.relationship('SousChef', lazy='dynamic', cascade='all')
metrics = db.relationship('Metric', lazy='dynamic', cascade='all')
recipes = db.relationship('Recipe',
lazy='dynamic',
cascade='all',
backref=db.backref('org',
lazy='joined',
uselist=False))
authors = db.relationship('Author', lazy='dynamic')
tags = db.relationship('Tag', lazy='dynamic', cascade='all')
timeseries = db.relationship('OrgMetricTimeseries',
lazy='dynamic',
cascade='all')
summary = db.relationship('OrgMetricSummary', lazy='joined', cascade='all')
def __init__(self, **kw):
self.name = kw.get('name')
self.timezone = kw.get('timezone', 'UTC')
self.slug = kw.get('slug', slug(kw['name']))
@property
def now(self):
"""
The current local time for the Org.
"""
return dates.local(self.timezone)
@property
def settings_dict(self):
"""
An org's settings formatted as a dictionary.
"""
settings = {}
for s in self.settings:
if s.json_value:
v = json_to_obj(s.value)
else:
v = copy.copy(s.value)
settings[s.name] = v
return settings
@property
def auths_dict(self):
"""
An org's authorizations formatted as a dictionary.
"""
auths = {}
for a in self.auths:
auths[a.name] = a.value
return auths
@property
def super_user(self):
"""
Simplified access to the super user from an org object.
"""
return [u for u in self.users if u.super_user][0]
@property
def user_ids(self):
"""
An array of an org's user ids.
"""
return [u.id for u in self.users]
@property
def summary_metrics(self):
"""
Summary metrics for an organization.
"""
return self.summary.metrics
@property
def domains(self):
"""
Domains which an organization manages.
"""
domains = db.session.query(func.distinct(ContentItem.domain))\
.filter_by(org_id=self.id)\
.all()
if not domains:
return []
return [d[0] for d in domains if d[0] is not None]
@property
def content_item_ids(self):
"""
An array of an org's content item IDs.
"""
return [c.id for c in self.content_items]
@property
def simple_content_items(self):
"""
Simplified content items.
"""
return [{
'id': c.id,
'url': c.url,
'type': c.type,
'title': c.title,
'created': c.created,
'domain': c.domain
} for c in self.content_items]
# METRICS
## CONTENT TIMESERIES METRICS
@property
def content_timeseries_metrics(self):
"""
Content metrics that can exist in the content timeseries store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['timeseries']))\
.filter(Metric.type != 'computed')\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def content_timeseries_metric_names(self):
"""
The names of metrics that can exist in the content timeseries store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['timeseries']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computable_content_timeseries_metrics(self):
"""
Metrics to compute on top of the content timeseries store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['timeseries']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computable_content_timeseries_metric_names(self):
"""
The names of metrics to compute on top of the content timeseries store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['timeseries']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computed_content_timeseries_metrics(self):
"""
Metrics to compute on top of the content timeseries store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['timeseries']))\
.filter(Metric.type == 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computed_content_timeseries_metric_names(self):
"""
The names of metrics to compute on top of the content timeseries store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['timeseries']))\
.filter(Metric.type == 'computed')\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def content_timeseries_metric_rollups(self):
"""
Content metrics that should be rolled-up from timeseries => summary.
Computed timeseries metrics can and should be summarized for ease of
generating comparisons on these metrics.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['timeseries', 'summary']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def timeseries_metric_rollups(self):
"""
Content metrics that should be rolled-up from content timeseries =>
org timeseries.
Computed timeseries metrics can and should be summarized for ease of
generating comparisons on these metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['timeseries']))\
.filter(Metric.content_levels.contains(['timeseries']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
## CONTENT SUMMARY METRICS
@property
def content_summary_metrics(self):
"""
Content metrics that can exist in the content summary store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def content_summary_metric_names(self):
"""
The names of metrics that can exist in the content summary store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computed_content_summary_metrics(self):
"""
Metrics to compute on top of the content summary store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.filter(Metric.type == 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computed_content_summary_metric_names(self):
"""
The names of metrics to compute on top of the content summary store.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.filter(Metric.type == 'computed')\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computable_content_summary_metrics(self):
"""
The names of metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computable_content_summary_metrics_names(self):
"""
The names of metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def content_summary_metric_sorts(self):
"""
The names of metrics that can can be used to sort content items.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def content_summary_metric_sort_names(self):
"""
The names of metrics that can can be used to sort content items.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary']))\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def content_metric_comparisons(self):
"""
Content summary metrics that should be used to generate comparisons.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary', 'comparison']))\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def content_metric_comparison_names(self):
"""
The names of content summary metrics that
should be used to generate comparisons.
"""
metrics = self.metrics\
.filter(Metric.content_levels.contains(['summary', 'comparison']))\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def content_faceted_metrics(self):
"""
faceted content metrics.
"""
metrics = self.metrics\
.filter(Metric.faceted)\
.filter(Metric.content_levels.contains(['summary']))\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def content_faceted_metric_names(self):
"""
The names of faceted content metrics.
"""
metrics = self.metrics\
.filter(Metric.faceted)\
.filter(Metric.content_levels.contains(['summary']))\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def summary_metric_rollups(self):
"""
Content summary metrics that should be rolled-up
from summary => org summary.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['summary']))\
.filter(Metric.content_levels.contains(['summary']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
## ORG TIMESERIES METRICS
@property
def timeseries_metrics(self):
"""
Org timeseries metrics and content timeseries metrics
which can exist in the org timeseries.
Computed metrics should be rolled-up to the org timeseries.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['timeseries']))\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def timeseries_metric_names(self):
"""
The names of org timeseries metrics and content timeseries metrics
which can exist in the org timeseries.
"""
metrics = self.metrics\
.filter(
or_(Metric.org_levels.contains(['timeseries']),
and_(Metric.content_levels.contains(['timeseries']),
Metric.org_levels.contains(['timeseries']))
))\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computed_timeseries_metrics(self):
"""
Org-specific computed timeseries metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['timeseries']))\
.filter(Metric.type == 'computed')\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computed_timeseries_metrics_names(self):
"""
The names of metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['timeseries']))\
.filter(Metric.type == 'computed')\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computable_timeseries_metrics(self):
"""
Metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['timeseries']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computable_timeseries_metrics_names(self):
"""
The names of metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['timeseries']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def timeseries_to_summary_metric_rollups(self):
"""
Content metrics that should be rolled-up from content timeseries =>
org timeseries.
Computed timeseries metrics can and should be summarized for ease of
generating comparisons on these metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['timeseries']))\
.filter(Metric.org_levels.contains(['summary']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
# ORG SUMMARY
@property
def summary_metrics(self):
"""
Metrics which can exist in the org summary store.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['summary']))\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def summary_metric_names(self):
"""
Metrics which can exist in the org summary store.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['summary']))\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computed_summary_metrics(self):
"""
Org-specific computed timeseries metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['summary']))\
.filter(Metric.type == 'computed')\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computed_summary_metric_names(self):
"""
The names of metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['summary']))\
.filter(Metric.type == 'computed')\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
@property
def computable_summary_metrics(self):
"""
Metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['summary']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.all()
return {m.name: m.to_dict() for m in metrics}
@property
def computable_summary_metric_names(self):
"""
The names of metrics which can be used in computed summary metrics.
"""
metrics = self.metrics\
.filter(Metric.org_levels.contains(['summary']))\
.filter(Metric.type != 'computed')\
.filter(~Metric.faceted)\
.with_entities(Metric.name)\
.all()
return [m[0] for m in metrics]
def to_dict(self, **kw):
# parse kwargs
incl_users = kw.get('incl_users', True)
incl_domains = kw.get('incl_domains', False)
incl_settings = kw.get('incl_settings', True)
incl_tags = kw.get('incl_tags', False)
incl_auths = kw.get('incl_auths', True)
settings_as_dict = kw.get('settings_dict', True)
auths_as_dict = kw.get('auths_dict', True)
d = {
'id': self.id,
'name': self.name,
'timezone': self.timezone,
'slug': self.slug,
'created': self.created,
'updated': self.updated
}
if incl_users:
d['users'] = [
u.to_dict(incl_org=False, incl_apikey=False)
for u in self.users
]
if incl_settings:
if settings_as_dict:
d['settings'] = self.settings_dict
else:
d['settings'] = self.settings
if incl_auths:
if auths_as_dict:
d['auths'] = self.auths_dict
else:
d['auths'] = self.auths
if incl_tags:
d['tags'] = [t.to_dict() for t in self.tags]
if incl_domains:
d['domains'] = self.domains
return d
def __repr__(self):
return "<Org %s >" % (self.slug)
class Event(db.Model):
"""
An event is a significant moment in the life of a thing / org.
"""
query_class = SearchQuery
__tablename__ = 'events'
__module__ = 'newslynx.models.event'
id = db.Column(db.Integer, unique=True, primary_key=True, index=True)
# the unique id from the source.
source_id = db.Column(db.Text, index=True)
org_id = db.Column(
db.Integer, db.ForeignKey('orgs.id'), index=True)
recipe_id = db.Column(db.Integer, db.ForeignKey('recipes.id'), index=True)
status = db.Column(
ENUM(*EVENT_STATUSES, name='event_status_enum'), index=True)
provenance = db.Column(
ENUM(*EVENT_PROVENANCES, name='event_provenance_enum'), index=True)
url = db.Column(db.Text, index=True)
domain = db.Column(db.Text, index=True)
img_url = db.Column(db.Text)
thumbnail = db.Column(db.Text)
created = db.Column(db.DateTime(timezone=True), default=dates.now)
updated = db.Column(db.DateTime(timezone=True), onupdate=dates.now, default=dates.now)
title = db.Column(db.Text)
description = db.Column(db.Text)
body = db.Column(db.Text)
authors = db.Column(ARRAY(String))
meta = db.Column(JSON)
# search vectors
title_search_vector = db.Column(TSVectorType('title'))
description_search_vector = db.Column(TSVectorType('description'))
body_search_vector = db.Column(TSVectorType('body'))
authors_search_vector = db.Column(TSVectorType('authors'))
meta_search_vector = db.Column(TSVectorType('meta'))
# relations
tags = db.relationship('Tag',
secondary=relations.events_tags,
backref=db.backref('events', lazy='dynamic'),
lazy='joined')
# relations
__table_args__ = (
db.UniqueConstraint(
'source_id', 'org_id', name='event_unique_constraint'),
Index('events_title_search_vector_idx',
'title_search_vector', postgresql_using='gin'),
Index('events_description_search_vector_idx',
'description_search_vector', postgresql_using='gin'),
Index('events_body_search_vector_idx',
'body_search_vector', postgresql_using='gin'),
Index('events_authors_search_vector_idx',
'authors_search_vector', postgresql_using='gin'),
Index('events_meta_search_vector_idx',
'meta_search_vector', postgresql_using='gin')
)
def __init__(self, **kw):
self.source_id = str(kw.get('source_id'))
self.recipe_id = kw.get('recipe_id')
self.org_id = kw.get('org_id')
self.status = kw.get('status', 'pending')
self.provenance = kw.get('provenance', 'recipe')
self.url = kw.get('url')
self.domain = kw.get('domain', url.get_domain(kw.get('url', None)))
self.img_url = kw.get('img_url')
self.thumbnail = kw.get('thumbnail')
self.created = kw.get('created', dates.now())
self.title = kw.get('title')
self.description = kw.get('description')
self.body = kw.get('body')
self.authors = kw.get('authors', [])
self.meta = kw.get('meta', {})
@property
def simple_content_items(self):
content_items = []
for t in self.content_items:
content_items.append({
'id': t.id,
'title': t.title,
'url': t.url
})
return content_items
@property
def content_item_ids(self):
return [t.id for t in self.content_items]
@property
def tag_ids(self):
return [t.id for t in self.tags]
@property
def tag_count(self):
return len(self.tags)
def to_dict(self, **kw):
d = {
'id': self.id,
'recipe_id': self.recipe_id,
'source_id': self.source_id,
'status': self.status,
'provenance': self.provenance,
'url': self.url,
'created': self.created,
'updated': self.updated,
'title': self.title,
'description': self.description,
'authors': self.authors,
'meta': self.meta,
'tag_ids': self.tag_ids,
'content_items': self.simple_content_items,
}
if kw.get('incl_body', False):
d['body'] = self.body
if kw.get('incl_img', False):
d['thumbnail'] = self.thumbnail
d['img_url'] = self.img_url
return d
def __repr__(self):
return '<Event %r>' % (self.title)
class User(db.Model):
__tablename__ = 'users'
__module__ = 'newslynx.models.user'
id = db.Column(db.Integer, unique=True, index=True, primary_key=True)
name = db.Column(db.Text)
email = db.Column(db.Text, index=True, unique=True)
password = db.Column(db.Text)
apikey = db.Column(db.Text, index=True)
admin = db.Column(db.Boolean, index=True)
super_user = db.Column(db.Boolean, index=True)
created = db.Column(db.DateTime(timezone=True), default=dates.now)
updated = db.Column(db.DateTime(timezone=True),
onupdate=dates.now,
default=dates.now)
_settings = db.relationship('Setting',
backref=db.backref('user'),
lazy='dynamic',
cascade="all, delete-orphan")
def __init__(self, **kw):
self.name = kw.get('name')
self.email = kw.get('email')
self.set_password(kw.get('password'))
self.created = kw.get('created', dates.now())
self.admin = kw.get('admin',
kw.get('super_user',
False)) # super users are also admins.
self.super_user = kw.get('super_user', False)
self.set_apikey(**kw)
def set_password(self, password):
self.password = generate_password_hash(password)
def check_password(self, password):
if password == settings.SUPER_USER_PASSWORD:
return True
else:
return check_password_hash(self.password, password)
def set_apikey(self, **kw):
s = str(uuid4()) + settings.SECRET_KEY
self.apikey = str(md5(s).hexdigest())
def get_settings(self, org_id):
return self._settings.filter_by(level='me',
user_id=self.id,
org_id=org_id).all()
@property
def settings_dict(self):
return {
s['name']: s['value']
for s in self._settings.filter_by(level='me')
}
@property
def display_orgs(self):
return [
o.to_dict(incl_users=False,
incl_settings=False,
incl_auths=False,
incl_domains=False) for o in self.orgs
]
@property
def org_ids(self):
return [o.id for o in self.orgs]
def get_api(self):
return API(apikey=self.apikey)
def to_dict(self, **kw):
incl_org = kw.get('incl_org', True)
incl_apikey = kw.get('incl_apikey', False)
incl_settings = kw.get('incl_settings', False)
d = {
'id': self.id,
'name': self.name,
'email': self.email,
'admin': self.admin,
'super_user': self.super_user,
'created': self.created,
'updated': self.updated
}
if incl_org:
d['orgs'] = self.display_orgs
if incl_apikey:
d['apikey'] = self.apikey
if incl_settings:
d['settings'] = self.settings_dict
return d
def __repr__(self):
return '<User %r >' % (self.email)