def event_delete_content_item(user, org, event_id, content_item_id):
"""
Remove a thing from an event.
"""
e = Event.query\
.filter_by(id=event_id, org_id=org.id)\
.first()
if not e:
raise NotFoundError(
'An Event with ID {} does not exist.'.format(event_id))
c = ContentItem.query\
.filter_by(id=content_item_id, org_id=org.id)\
.first()
if not c:
raise RequestError(
'A ContentItem with ID {} does not exist.'.format(content_item_id))
if content_item_id not in e.content_item_ids:
raise RequestError(
'An Event with ID {} does not currently have an association '
'with a ContentItem with ID {}'.format(event_id, content_item_id))
# remove the content item form the event.
e.content_items.remove(c)
db.session.add(e)
db.session.commit()
# return modified event
return jsonify(e)
def cook_a_recipe(user, org, recipe_id):
"""
Run recipes via the API.
"""
r = fetch_by_id_or_field(Recipe, 'slug', recipe_id, org_id=org.id)
if not r:
raise RequestError(
'Recipe with id/slug {} does not exist.'.format(recipe_id))
# setup kwargs for merlynne
kw = dict(org=org.to_dict(incl_auths=True,
auths_as_dict=True,
settings_as_dict=True,
incl_domains=True,
incl_users=True),
apikey=user.apikey,
recipe=r.to_dict(),
recipe_obj=r,
sous_chef_path=r.sous_chef.runs)
# cook recipe
merlynne = Merlynne(**kw)
try:
job_id = merlynne.cook_recipe()
except Exception as e:
raise RequestError(
'There was a problem initializing the SousChef: {}'.format(
e.message))
# # return job status url
ret = url_for_job_status(apikey=user.apikey, job_id=job_id, queue='recipe')
return jsonify(ret, status=202)
def event_add_thing(user, org, event_id, content_item_id):
"""
Add a thing to an event.
"""
e = Event.query.filter_by(id=event_id, org_id=org.id).first()
if not e:
raise NotFoundError(
'An Event with ID {} does not exist.'.format(event_id))
if not e.status == 'approved':
raise RequestError(
'You must first approve an Event before adding additional ContentItems.'
)
c = ContentItem.query\
.filter_by(id=content_item_id, org_id=org.id)\
.first()
if not c:
raise RequestError(
'A ContentItem with ID {} does not exist.'.format(content_item_id))
# add content item to event
if c.id not in e.content_item_ids:
e.content_items.append(c)
db.session.add(e)
db.session.commit()
# return modified event
return jsonify(e)
def create_author(user, org):
"""
Create an author.
"""
req_data = request_data()
cols = get_table_columns(Author)
if 'name' not in req_data:
raise RequestError("A 'name' is required to create an Author.")
for k in req_data.keys():
if k not in cols or k in ['id', 'org_id']:
req_data.pop(k, None)
# upper-case.
elif k == 'name':
req_data[k] = req_data[k].upper()
a = Author(org_id=org.id, **req_data)
try:
db.session.add(a)
db.session.commit()
except Exception as e:
raise RequestError(
'There was an error creating this Author: {}'.format(e.message))
return jsonify(a)
def event_add_tag(user, org, event_id, tag_id):
"""
Add a tag to an event.
"""
e = Event.query\
.filter_by(id=event_id, org_id=org.id)\
.first()
if not e:
raise NotFoundError(
'An Event with ID {} does not exist.'.format(event_id))
if not e.status == 'approved':
raise RequestError(
'You must first approve an Event before adding additional Tags.')
tag = Tag.query\
.filter_by(id=tag_id, org_id=org.id)\
.first()
if not tag:
raise NotFoundError('Tag with ID {} does not exist.'.format(tag_id))
if tag.type != 'impact':
raise RequestError('Events can only be assigned Impact Tags.')
if tag.id not in e.tag_ids:
e.tags.append(tag)
db.session.add(e)
db.session.commit()
# return modified event
return jsonify(e)
def org_add_user(user, org_id_slug, user_email):
if not user.admin:
raise AuthError('You must be an admin to add a user to an Org.')
# fetch org
org = fetch_by_id_or_field(Org, 'slug', org_id_slug)
if not org:
raise NotFoundError('This Org does not exist.')
# ensure the active user can edit this Org
if user.id not in org.user_ids:
raise ForbiddenError('You are not allowed to edit this Org.')
# localize
localize(org)
# get this new user by id / email
new_org_user = fetch_by_id_or_field(User, 'email', user_email)
if not new_org_user:
raise RequestError('User "{}" does not exist'.format(user_email))
# ensure that user is not already a part of this Org.
if new_org_user.id in org.user_ids:
raise RequestError('User "{}" is already a part of Org "{}"'.format(
new_org_user.email, org.name))
org.users.append(new_org_user)
db.session.commit()
return jsonify(new_org_user)
def prepare_metrics(obj,
org_metric_lookup,
valid_levels=[],
parent_obj='ContentItem',
check_timeseries=True):
"""
Validate a metric.
"""
# check if metrics exist and are properly formatted.
obj.update(obj.pop('metrics', {}))
for k in obj.keys():
m = org_metric_lookup.get(k)
if not m:
raise RequestError(
"Metric '{}' does not exist at this level.".format(k))
if m['faceted'] and not isinstance(obj[k], list):
raise RequestError(
"Metric '{}' is faceted but was not passed in as a list.".
format(k))
if m['faceted'] and not set(
obj[k][0].keys()) == set(METRIC_FACET_KEYS):
raise RequestError(
"Metric '{}' is faceted, but it\'s elements are not properly formatted. "
"Each facet must be a dictionary of '{\"facet\":\"facet_name\", \"value\": 1234}"
.format(k))
# parse number
obj[k] = stats.parse_number(obj[k])
return obj
def create_template(user, org):
# get the request data
req_data = request_data()
name = req_data.get('name')
slug = req_data.get('slug')
template = req_data.get('template')
format = req_data.get('format')
if not name or not template or not format:
raise RequestError(
"You must pass in a 'name', 'format', and 'template' to create a template. "
"You only passed in: {}".format(", ".join(req_data.keys())))
try:
t = Tmpl(template)
except Exception as e:
raise RequestError('This template is invalid: {}'.format(e.message))
t = Template(org_id=org.id, name=name, template=template, format=format)
if slug:
t.slug = slug
db.session.add(t)
# no duplicates.
try:
db.session.commit()
except Exception as e:
raise RequestError(e.message)
return jsonify(t)
def create_recipe(user, org):
req_data = request_data()
sous_chef = req_data.pop('sous_chef', arg_str('sous_chef', None))
if not sous_chef:
raise RequestError(
'You must pass in a SousChef ID or slug to create a recipe.')
sc = fetch_by_id_or_field(SousChef, 'slug', sous_chef)
if not sc:
raise RequestError(
'A SousChef does not exist with ID/slug {}'.format(sous_chef))
# validate the recipe and add it to the database.
recipe = recipe_schema.validate(req_data, sc.to_dict())
r = Recipe(sc, user_id=user.id, org_id=org.id, **recipe)
db.session.add(r)
db.session.commit()
# if the recipe creates metrics create them in here.
if 'metrics' in sc.creates:
for name, params in sc.metrics.items():
m = Metric(name=name, recipe_id=r.id, org_id=org.id, **params)
db.session.add(m)
try:
db.session.commit()
except Exception as e:
raise ConflictError(
"You tried to create a metric that already exists. "
"Here's the exact error:\n{}".format(e.message))
return jsonify(r)
def bulk_create_content_summary(user, org):
"""
bulk upsert summary metrics for an organization's content items.
"""
req_data = request_data()
# check for valid format.
if not isinstance(req_data, list):
raise RequestError(
"Bulk endpoints require a list of json objects."
)
# check for content_item_id.
if not 'content_item_id' in req_data[0].keys():
raise RequestError(
'You must pass in a content_item_id with each record.')
job_id = ingest_bulk.content_summary(
req_data,
org_id=org.id,
metrics_lookup=org.content_summary_metrics,
content_item_ids=org.content_item_ids,
commit=False)
ret = url_for_job_status(apikey=user.apikey, job_id=job_id, queue='bulk')
return jsonify(ret, status=202)
def update_tag(user, org, tag_id):
"""
Update an individual tag.
"""
# fetch the tag object
tag = fetch_by_id_or_field(Tag, 'slug', tag_id, org_id=org.id)
if not tag:
raise NotFoundError(
'A Tag with ID {} does not exist'
.format(tag_id))
# fetch the request data.
req_data = request_data()
# check hex code
if 'color' in req_data:
validate_hex_code(req_data['color'])
# check tag type
if 'type' in req_data:
validate_tag_types(req_data['type'])
# if tag type is "impact" ensure a proper category and
# level are included
if req_data['type'] == 'impact':
validate_tag_categories(req_data['category'])
validate_tag_levels(req_data['level'])
# check if levels + categories are being assigned to
# subject tags
if tag.type == 'subject':
if req_data.get('category') or req_data.get('level'):
raise RequestError(
'Categories and Levels can only be set for Impact Tags')
# set org id
req_data['org_id'] = org.id
# filter out non-table columns
columns = get_table_columns(Tag)
for k in req_data.keys():
if k not in columns:
req_data.pop(k)
# update attributes
for k, v in req_data.items():
setattr(tag, k, v)
db.session.add(tag)
# check for dupes
try:
db.session.commit()
except Exception as err:
raise RequestError(err.message)
return jsonify(tag)
def _check_content_item_id(obj, content_item_ids):
"""
Raise errors if content item is missing.
"""
cid = obj.pop('content_item_id', None)
if not cid:
raise RequestError('Object is missing a "content_item_id"')
if not cid in content_item_ids:
raise RequestError('Content Item with ID {} doesnt exist'.format(cid))
return cid
def create_setting(user, org, level):
if level not in ['me', 'orgs']:
raise NotFoundError(
'You cannot store settings for \'{}\''.format(level))
# get the request data
req_data = request_data()
name = req_data.get('name')
value = req_data.get('value')
json_value = req_data.get('json_value', False)
if not name or not value:
raise RequestError(
"You must pass in a 'name' and 'value' to create a setting. "
"You only passed in: {}".format(", ".join(req_data.keys())))
# if it's a json_value check whether we can parse it as such
if json_value:
if isinstance(value, basestring):
try:
json_to_obj(value)
except:
raise RequestError(
"Setting '{}' with value '{}' was declared as a "
"'json_value' but could not be parsed as such.".format(
name, value))
s = Setting(org_id=org.id,
user_id=user.id,
level=level,
name=name,
value=value,
json_value=json_value or False)
db.session.add(s)
# no duplicates.
try:
db.session.commit()
except Exception as e:
raise ConflictError(e.message)
# temporary hack for 'timezone' setting in the APP.
if 'name' == 'timezone' and level == 'orgs':
org.timezone = value
try:
db.session.add(org)
db.session.commit()
except Exception as e:
raise RequestError(
"An error occurred while updating the timezone. "
"Here's the error message: {}".format(org.name, e.message))
return jsonify(s)
def content_timeseries(obj,
org_id=None,
metrics_lookup=None,
content_item_ids=None,
commit=True):
"""
Ingest Timeseries Metrics for a content item.
"""
# if not content_item_id or not org or not metrics_lookup:
# raise RequestError('Missing required kwargs.')
content_item_id = obj.pop('content_item_id')
if not content_item_id:
raise RequestError('Object is missing a "content_item_id"')
if not content_item_id in content_item_ids:
raise RequestError(
'Content Item with ID {} doesnt exist'.format(content_item_id))
cmd_kwargs = {"org_id": org_id, "content_item_id": content_item_id}
# parse datetime.
if 'datetime' not in obj:
cmd_kwargs['datetime'] = dates.floor_now(unit='hour',
value=1).isoformat()
else:
ds = obj.pop('datetime')
dt = dates.parse_iso(ds)
cmd_kwargs['datetime'] = dates.floor(dt, unit='hour',
value=1).isoformat()
metrics = ingest_util.prepare_metrics(obj,
metrics_lookup,
valid_levels=['content_item', 'all'],
check_timeseries=True)
# upsert command
cmd = """SELECT upsert_content_metric_timeseries(
{org_id},
{content_item_id},
'{datetime}',
'{metrics}')
""".format(metrics=obj_to_json(metrics), **cmd_kwargs)
if commit:
try:
db.session.execute(cmd)
except Exception as err:
raise RequestError(err.message)
cmd_kwargs['metrics'] = metrics
return cmd
def author_add_content(user, org, author_id, content_item_id):
"""
Add an author to a content item.
"""
a = Author.query\
.filter_by(id=author_id, org_id=org.id)\
.first()
if not a:
raise NotFoundError(
'Author with ID "{}" does not exist."'.format(author_id))
c = ContentItem.query\
.filter_by(id=content_item_id, org_id=org.id)\
.first()
if not c:
raise RequestError(
'ContentItem with ID {} does not exist.'.format(content_item_id))
if a.id not in c.author_ids:
c.authors.append(a)
db.session.add(c)
db.session.commit()
# return modified event
return jsonify(a.to_dict(incl_content=True))
def event_delete_tag(user, org, event_id, tag_id):
"""
Remove a tag from an event.
"""
e = Event.query\
.filter_by(id=event_id, org_id=org.id)\
.first()
if not e:
raise NotFoundError(
'An Event with ID {} does not exist.'.format(event_id))
if tag_id not in e.tag_ids:
raise RequestError(
'An Event with ID {} does not currently have an association '
'with a Tag with ID {}.'.format(event_id, tag_id))
# remove tag from event
for tag in e.tags:
if tag.id == tag_id:
e.tags.remove(tag)
# update metrics associated with these content item ids
if len(e.content_item_ids):
rollup_metric.content_summary_from_events(org, e.content_item_ids)
db.session.add(e)
db.session.commit()
# return modified event
return jsonify(e)
def extract(user):
url = arg_str('url', default=None)
type = arg_str('type', default='article')
force_refresh = arg_bool('force_refresh', default=False)
format = arg_str('format', default='json')
if not url:
raise RequestError("A url is required.")
if force_refresh:
extract_cache.debug = True
cr = extract_cache.get(url, type)
if not cr:
extract_cache.invalidate(url, type)
raise InternalServerError('Something went wrong. Try again.')
resp = {
'cache': cr,
'data': cr.value
}
if format == 'html':
return render_template(
'extract_preview.html',
data=resp)
return jsonify(resp)
def _provenance(obj, recipe, type='event'):
"""
Determine provenance for events or content items.
Handle source ids for events.
"""
if not recipe:
obj['provenance'] = 'manual'
obj['recipe_id'] = None
if type == 'event':
src_id = obj.get('source_id')
if not src_id:
src_id = gen_uuid()
obj['source_id'] = "manual:{}".format(src_id)
else:
if type == 'event':
# recipe-generated events must pass in a source id
if 'source_id' not in obj:
raise RequestError(
'Recipe generated events must include a source_id.')
# reformant source id.
obj['source_id'] = "{}:{}"\
.format(str(recipe.slug), str(obj['source_id']))
obj['provenance'] = 'recipe'
obj['recipe_id'] = recipe.id
return obj
def event_delete_tag(user, org, event_id, tag_id):
"""
Remove a tag from an event.
"""
e = Event.query\
.filter_by(id=event_id, org_id=org.id)\
.first()
if not e:
raise NotFoundError(
'An Event with ID {} does not exist.'.format(event_id))
if tag_id not in e.tag_ids:
raise RequestError(
'An Event with ID {} does not currently have an association '
'with a Tag with ID {}.'.format(event_id, tag_id))
# remove tag from event
for tag in e.tags:
if tag.id == tag_id:
e.tags.remove(tag)
db.session.add(e)
db.session.commit()
# return modified event
return jsonify(e)
def delete_recipe(user, org, recipe_id):
r = fetch_by_id_or_field(Recipe, 'slug', recipe_id, org_id=org.id)
if not r:
raise RequestError(
'Recipe with id/slug {} does not exist.'.format(recipe_id))
force = arg_bool('force', default=False)
if force:
db.session.delete(r)
else:
# just delete recipes with no approved events.
event_cnt = r.events.filter_by(status='approved').count()
if event_cnt > 0:
r.status = 'inactive'
db.session.add(r)
else:
db.session.delete(r)
# set the status of associated events to 'deleted'
cmd = """
UPDATE events SET status='deleted' WHERE recipe_id = {} AND status='pending';
""".format(r.id)
db.session.execute(cmd)
db.session.commit()
return delete_response()
def update_recipe(user, org, recipe_id):
r = fetch_by_id_or_field(Recipe, 'slug', recipe_id, org_id=org.id)
if not r:
raise RequestError(
'Recipe with id/slug {} does not exist.'.format(recipe_id))
# fetch request date and update / validate.
req_data = request_data()
new_recipe = recipe_schema.update(r, req_data, r.sous_chef.to_dict())
# if the requesting user hasn't initializied this recipe,
# do it for them:
status = new_recipe.get('status', 'uninitialized')
if r.status == 'uninitialized' and status == 'uninitialized':
new_recipe['status'] = 'stable'
# update pickled options
new_opts = new_recipe.pop('options')
r.set_options(new_opts)
# update all other fields
for col, val in new_recipe.items():
setattr(r, col, val)
db.session.add(r)
db.session.commit()
return jsonify(r)
def author_remove_content(user, org, author_id, content_item_id):
"""
Remove an author to a content item.
"""
a = Author.query\
.filter_by(id=author_id, org_id=org.id)\
.first()
if not a:
raise NotFoundError(
'Author with ID "{}" does not exist."'.format(author_id))
c = ContentItem.query\
.filter_by(id=content_item_id, org_id=org.id)\
.first()
if not c:
raise RequestError(
'ContentItem with ID {} does not exist.'.format(content_item_id))
if a.id in c.author_ids:
a.content_items.remove(c)
db.session.add(a)
db.session.commit()
return delete_response()
def org_user(user, org_id_slug, user_email):
# fetch org
org = fetch_by_id_or_field(Org, 'slug', org_id_slug)
if not org:
raise NotFoundError('This Org does not exist.')
# ensure the active user can edit this Org
if user.id not in org.user_ids:
raise ForbiddenError('You are not allowed to access this Org')
# localize
localize(org)
# get this new user by id / email
org_user = fetch_by_id_or_field(User, 'email', user_email)
if not org_user:
raise RequestError('This user does not yet exist')
# check whether this user can view this other user:
if not len(list(set(org_user.org_ids).intersection(set(user.org_ids)))):
raise ForbiddenError('You are not allowed to view this user.'.format(
user.email))
return jsonify(org_user)
def update_metric(user, org, name_id):
m = fetch_by_id_or_field(Metric, 'name', name_id, org_id=org.id)
if not m:
raise NotFoundError(
'Metric "{}" does not yet exist for Org "{}"'
.format(name_id, org.name))
# get the request data
req_data = request_data()
# filter out any non-columns
columns = get_table_columns(Metric)
for k in req_data.keys():
if k not in columns:
req_data.pop(k)
# don't ever overwrite these:
for k in ['id', 'recipe_id', 'name', 'org_id', 'created', 'updated']:
if k in req_data:
req_data.pop(k, None)
# update fields
for k, v in req_data.items():
setattr(m, k, v)
try:
db.session.add(m)
db.session.commit()
except Exception as e:
raise RequestError("Error updating Metric: {}".format(e.message))
return jsonify(m)
def author_remove_content(user, org, author_id, content_item_id):
"""
Remove an author from a content item.
"""
a = fetch_by_id_or_field(Author,
'name',
author_id,
org_id=org.id,
transform='upper')
if not a:
raise NotFoundError(
'Author with ID/Name "{}" does not exist."'.format(author_id))
c = ContentItem.query\
.filter_by(id=content_item_id, org_id=org.id)\
.first()
if not c:
raise RequestError(
'ContentItem with ID {} does not exist.'.format(content_item_id))
if a.id in c.author_ids:
a.content_items.remove(c)
db.session.add(a)
db.session.commit()
return delete_response()
def listify_data_arg(name):
"""
Allow for multiple list formats of
data args including:
- comma-separated list of ids
- list of ids
- list of dicts with an id key
"""
value = request_data().get(name)
if not value:
return []
if not isinstance(value, list):
if ',' in value:
value = [v.strip() for v in value.split(',')]
else:
value = [value]
if isinstance(value[0], dict):
try:
value = [a['id'] for a in value]
except KeyError:
raise RequestError(
"When passing in a dictionary or list of dictionaries "
"to '{}', they must contain a key with the name 'id'".format(
name))
return value
def _content_item_provenance(obj, org_id):
"""
if there's not a recipe_id set the provenance as "manual"
otherwise check it the recipe id is valid and set as "recipe"
"""
# this is a manual upload
if 'recipe_id' not in obj or not obj['recipe_id']:
obj['provenance'] = 'manual'
# this is from a recipe
else:
# fetch the associated recipe
r = Recipe.query\
.filter_by(id=obj['recipe_id'])\
.filter_by(org_id=org_id)\
.first()
if not r:
raise RequestError(
'Recipe id "{recipe_id}" does not exist.'.format(**obj))
# set this event as non-manual
obj['provenance'] = 'recipe'
return obj
def create_content_item_timeseries(user, org, content_item_id):
"""
Upsert content timseries metrics.
"""
c = ContentItem.query\
.filter_by(id=content_item_id)\
.filter_by(org_id=org.id)\
.first()
if not c:
raise NotFoundError(
'A ContentItem with ID {} does not exist'
.format(content_item_id))
req_data = request_data()
# check for valid format.
if not isinstance(req_data, dict):
raise RequestError(
"Non-bulk endpoints require a single json object."
)
# insert content item id
req_data['content_item_id'] = content_item_id
ret = ingest_metric.content_timeseries(
req_data,
org_id=org.id,
metrics_lookup=org.content_timeseries_metrics,
commit=True)
return jsonify(ret)
def create_tag(user, org):
"""
Create a tag.
"""
req_data = request_data()
# check for required keys
for k in ['name', 'type', 'color']:
if k not in req_data:
raise RequestError(
'A Tag requires a "name", "color", and "type"')
# check hex code
validate_hex_code(req_data['color'])
# check tag type
validate_tag_types(req_data['type'])
# if tag type is "impact" ensure a proper category and level are included
if req_data['type'] == 'impact':
for k in ['level', 'category']:
if k not in req_data:
raise RequestError(
'An Impact Tag requires a "level" and "category"')
validate_tag_categories(req_data['category'])
validate_tag_levels(req_data['level'])
elif req_data['type'] == 'subject':
for k in ['level', 'category']:
if k in req_data:
raise RequestError(
'Categories and Levels can only be set for Impact Tags.')
# create the tag
tag = Tag(org_id=org.id, **req_data)
db.session.add(tag)
# check for dupes
try:
db.session.commit()
except Exception as err:
raise ConflictError(err.message)
return jsonify(tag)
def get_recipe(user, org, recipe_id):
r = fetch_by_id_or_field(Recipe, 'slug', recipe_id, org_id=org.id)
if not r:
raise RequestError(
'Recipe with id/slug {} does not exist.'.format(recipe_id))
# add in event counts.
return jsonify(r)
