def _save_user_data_dir(u, user_mc):
# make a dir first (prefix with user_id for extra security)
temp_dir = tempfile.mkdtemp(prefix='user{}'.format(u.profile['auth_users_id']))
# user profile
with open(os.path.join(temp_dir, 'profile.json'), 'w') as outfile:
profile = u.profile
json.dump(profile, outfile)
# topic-level permissions
with open(os.path.join(temp_dir, 'topic-permissions.csv'), 'w') as outfile:
topics = user_mc.topicList(limit=1000)['topics']
user_owned_topics = topics_user_owns(topics, u.profile['email'])
topic_permission_list = [{
'topics_id': t['topics_id'],
'topic_name': t['name'],
'permission': t['user_permission'],
} for t in user_owned_topics]
writer = csv.DictWriter(outfile, ['topics_id', 'topic_name', 'permission'])
writer.writeheader()
writer.writerows(topic_permission_list)
# saved searches
with open(os.path.join(temp_dir, 'saved-searches.json'), 'w') as outfile:
search_list = user_db.get_users_lists(u.name, 'searches')
json.dump(search_list, outfile)
# starred sources
with open(os.path.join(temp_dir, 'starred-sources.csv'), 'w') as outfile:
user_favorited = user_db.get_users_lists(user_name(), 'favoriteSources')
media_sources = [user_mc.media(media_id) for media_id in user_favorited]
media_sources = [{
'media_id': m['media_id'],
'name': m['name'],
'url': m['url']
} for m in media_sources]
writer = csv.DictWriter(outfile, ['media_id', 'name', 'url'])
writer.writeheader()
writer.writerows(media_sources)
# starred collections
with open(os.path.join(temp_dir, 'starred-collections.csv'), 'w') as outfile:
user_favorited = user_db.get_users_lists(user_name(), 'favoriteCollections')
collections = [user_mc.tag(tags_id) for tags_id in user_favorited]
collections = [{
'tags_id': c['tags_id'],
'label': c['label'],
'description': c['description']
} for c in collections]
writer = csv.DictWriter(outfile, ['tags_id', 'label', 'description'])
writer.writeheader()
writer.writerows(collections)
# starred topics
with open(os.path.join(temp_dir, 'starred-topics.csv'), 'w') as outfile:
user_favorited = user_db.get_users_lists(user_name(), 'favoriteTopics')
topics = [user_mc.topic(topics_id) for topics_id in user_favorited]
topics = [{
'topics_id': t['topics_id'],
'name': t['name'],
'description': t['description']
} for t in topics]
writer = csv.DictWriter(outfile, ['topics_id', 'name', 'description'])
writer.writeheader()
writer.writerows(topics)
return temp_dir
def source_suggest():
user_mc = user_admin_mediacloud_client()
url = request.form['url']
feed_url = request.form['feedurl'] if 'feedurl' in request.form else None
name = request.form['name'] if 'name' in request.form else None
reason = request.form['reason'] if 'reason' in request.form else None
tag_ids_to_add = tag_ids_from_collections_param()
new_suggestion = user_mc.mediaSuggest(url=url,
name=name,
feed_url=feed_url,
reason=reason,
collections=tag_ids_to_add)
# send an email confirmation
email_title = "Thanks for Suggesting " + url
send_html_email(
email_title, [user_name(), '*****@*****.**'],
render_template("emails/source_suggestion_ack.txt",
username=user_name(),
name=name,
url=url,
feed_url=feed_url,
reason=reason),
render_template("emails/source_suggestion_ack.html",
username=user_name(),
name=name,
url=url,
feed_url=feed_url,
reason=reason))
# and return that it worked
return jsonify(new_suggestion)
def source_suggestion_update(suggestion_id):
user_mc = user_admin_mediacloud_client()
suggestion = _media_suggestion(user_mc, suggestion_id)
if suggestion is None:
return json_error_response("Unknown suggestion id {}".format(suggestion_id))
status = request.form['status']
reason = request.form['reason']
results = None
email_note = ""
if status == "approved":
# if approved, we have to create it
flattend_tags = [t['tags_id'] for t in suggestion['tags_ids']]
media_source_to_create = {
'url': suggestion['url'],
'name': suggestion['name'],
'feeds': [suggestion['feed_url']],
'tags_ids': flattend_tags,
'editor_notes': 'Suggested approved by {} on because {}. Suggested by {} on {} because {}'
'(id #{}).'.format(user_name(), datetime.now().strftime("%I:%M%p on %B %d, %Y"), reason,
suggestion['email'], suggestion['date_submitted'], suggestion['reason'],
suggestion['media_suggestions_id']
)
}
creation_results = user_mc.mediaCreate([media_source_to_create])[0]
if creation_results['status'] == 'error':
status = "pending" # so the email update looks good.
email_note = creation_results['error']+". "
else:
email_note = "This source is "+str(creation_results['status'])+". "
results = user_mc.mediaSuggestionsMark(suggestion_id, status, reason, creation_results['media_id'])
else:
# if rejected just mark it as such
results = user_mc.mediaSuggestionsMark(suggestion_id, status, reason)
# send an email to the person that suggested it
url = suggestion['url']
email_title = "Source Suggestion {}: {}".format(status, url)
content_title = "We {} {}".format(status, url)
content_body = "Thanks for the suggestion. {}{}".format(email_note, reason)
action_text = "Login to Media Cloud"
action_url = "https://sources.mediacloud.org/#/login"
# send an email confirmation
send_html_email(email_title,
[user_name(), '*****@*****.**'],
render_template("emails/generic.txt", content_title=content_title, content_body=content_body,
action_text=action_text, action_url=action_url),
render_template("emails/generic.html", email_title=email_title, content_title=content_title,
content_body=content_body, action_text=action_text, action_url=action_url)
)
# and return that it worked or not
if status == "pending":
return json_error_response(email_note)
return jsonify(results)
def source_suggestion_update(suggestion_id):
user_mc = user_admin_mediacloud_client()
suggestion = _media_suggestion(user_mc, suggestion_id)
if suggestion is None:
return json_error_response("Unknown suggestion id {}".format(suggestion_id))
status = request.form['status']
reason = request.form['reason']
results = None
email_note = ""
if status == "approved":
# if approved, we have to create it
flattend_tags = [t['tags_id'] for t in suggestion['tags_ids']]
media_source_to_create = { 'url': suggestion['url'],
'name': suggestion['name'],
'feeds': [suggestion['feed_url']],
'tags_ids': flattend_tags,
'editor_notes': 'Suggested approved by {} on because {}. Suggested by {} on {} because {} (id #{}).'.format(
user_name(), datetime.now().strftime("%I:%M%p on %B %d, %Y"), reason,
suggestion['email'], suggestion['date_submitted'], suggestion['reason'], suggestion['media_suggestions_id']
)
}
creation_results = user_mc.mediaCreate([media_source_to_create])[0]
if creation_results['status'] == 'error':
status = "pending" # so the email update looks good.
email_note = creation_results['error']+". "
else:
email_note = "This source is "+str(creation_results['status'])+". "
results = user_mc.mediaSuggestionsMark(suggestion_id, status, reason, creation_results['media_id'])
else:
# if rejected just mark it as such
results = user_mc.mediaSuggestionsMark(suggestion_id, status, reason)
# send an email to the person that suggested it
url = suggestion['url']
email_title = "Source Suggestion {}: {}".format(status, url)
content_title = "We {} {}".format(status, url)
content_body = "Thanks for the suggestion. {}{}".format(email_note, reason)
action_text = "Login to Media Cloud"
action_url = "https://sources.mediacloud.org/#/login"
# send an email confirmation
send_html_email(email_title,
[user_name(), '*****@*****.**'],
render_template("emails/generic.txt",
content_title=content_title, content_body=content_body, action_text=action_text, action_url=action_url),
render_template("emails/generic.html",
email_title=email_title, content_title=content_title, content_body=content_body, action_text=action_text, action_url=action_url)
)
# and return that it worked or not
if status == "pending":
return json_error_response(email_note)
return jsonify(results)
def topic_favorites():
user_mc = user_mediacloud_client()
favorite_topic_ids = user_db.get_users_lists(user_name(), 'favoriteTopics')
favorited_topics = [user_mc.topic(tid) for tid in favorite_topic_ids]
for t in favorited_topics:
t['isFavorite'] = True
return jsonify({'topics': favorited_topics})
def topic_filter_cascade_list():
public_topics = sorted_public_topic_list()
# for t in sorted_public_topics:
# t['detailInfo'] = get_topic_info_per_snapshot_timespan(t['topics_id'])
# check if user had favorites or personal
user_topics = []
favorited_topics = []
results = {'link_ids': []}
if is_user_logged_in():
user_mc = user_admin_mediacloud_client()
link_id = request.args.get('linkId')
results = user_mc.topicList(link_id=link_id)
user_topics = results['topics']
favorite_topic_ids = db.get_users_lists(user_name(), 'favorite'
'Topics')
# mark all the public topics as favorite or not
for t in public_topics:
t['isFavorite'] = t['topics_id'] in favorite_topic_ids
# mark all the user's topics as favorite or not
for t in user_topics:
t['isFavorite'] = t['topics_id'] in favorite_topic_ids
# fill in the list of favorite topics (need to hit server because they might no be in the other results)
favorited_topics = [user_mc.topic(tid) for tid in favorite_topic_ids]
for t in favorited_topics:
t['isFavorite'] = True
return jsonify({
'topics': {
'favorite': favorited_topics,
'personal': user_topics,
'public': public_topics
},
'link_ids': results['link_ids']
})
def _email_batch_source_update_results(audit_feedback):
email_title = "Source Batch Updates"
content_title = "You just uploaded {} sources to a collection.".format(
len(audit_feedback))
updated_sources = []
for updated in audit_feedback:
updated_sources.append(updated)
content_body = updated_sources
action_text = "Login to Media Cloud"
action_url = "https://sources.mediacloud.org/#/login"
# send an email confirmation
send_html_email(
email_title, [user_name(), '*****@*****.**'],
render_template("emails/source_batch_upload_ack.txt",
content_title=content_title,
content_body=content_body,
action_text=action_text,
action_url=action_url),
render_template("emails/source_batch_upload_ack.html",
email_title=email_title,
content_title=content_title,
content_body=content_body,
action_text=action_text,
action_url=action_url))
def source_stats(media_id):
username = user_name()
results = {}
# story count
media_query = "(media_id:{})".format(media_id)
source_specific_story_count = apicache.source_story_count(media_query)
results['story_count'] = source_specific_story_count
# health
media_health = _cached_media_source_health(username, media_id)
results['num_stories_90'] = media_health[
'num_stories_90'] if 'num_stories_90' in media_health else None
results['start_date'] = media_health[
'start_date'] if 'start_date' in media_health else None
info = _media_source_details(media_id)
user_can_see_private_collections = user_has_auth_role(ROLE_MEDIA_EDIT)
visible_collections = [
c for c in info['media_source_tags']
if ((c['tag_sets_id'] in VALID_COLLECTION_TAG_SETS_IDS) and (
(c['show_on_media'] == 1) or user_can_see_private_collections))
]
results['collection_count'] = len(visible_collections)
# geography tags
results['geoPct'] = apicache.tag_coverage_pct(media_query,
TAG_SET_GEOCODER_VERSION)
# nyt theme
results['nytPct'] = apicache.tag_coverage_pct(media_query,
TAG_SET_NYT_LABELS_VERSION)
return jsonify(results)
def favorite_sources():
user_mc = user_mediacloud_client()
user_favorited = db.get_users_lists(user_name(), 'favoriteSources')
favorited_s = [user_mc.media(media_id) for media_id in user_favorited]
for s in favorited_s:
s['isFavorite'] = True
return jsonify({'list': favorited_s})
def favorite_collections():
user_mc = user_mediacloud_client()
user_favorited = db.get_users_lists(user_name(), 'favoriteCollections')
favorited_collections = [user_mc.tag(tag_id) for tag_id in user_favorited]
for s in favorited_collections:
s['isFavorite'] = True
return jsonify({'list': favorited_collections})
def favorite_topics():
user_mc = user_admin_mediacloud_client()
user_favorited = db.get_users_lists(user_name(), 'favoriteTopics')
favorited_topics = [user_mc.topic(topic_id) for topic_id in user_favorited]
for t in favorited_topics:
t['isFavorite'] = True
t['detailInfo'] = get_topic_info_per_snapshot_timespan(t['topics_id'])
return jsonify({'topics': favorited_topics})
def collection_set_favorited(collection_id):
favorite = request.form["favorite"]
username = user_name()
if int(favorite) == 1:
user_db.add_item_to_users_list(username, 'favoriteCollections', int(collection_id))
else:
user_db.remove_item_from_users_list(username, 'favoriteCollections', int(collection_id))
return jsonify({'isFavorite': favorite})
def source_set_favorited(media_id):
favorite = request.form["favorite"]
username = user_name()
if int(favorite) == 1:
user_db.add_item_to_users_list(username, 'favoriteSources', int(media_id))
else:
user_db.remove_item_from_users_list(username, 'favoriteSources', int(media_id))
return jsonify({'isFavorite': favorite})
def collection_set_favorited(collection_id):
favorite = request.form["favorite"]
username = user_name()
if int(favorite) == 1:
user_db.add_item_to_users_list(username, 'favoriteCollections', int(collection_id))
else:
user_db.remove_item_from_users_list(username, 'favoriteCollections', int(collection_id))
return jsonify({'isFavorite': favorite})
def topic_favorites():
user_mc = user_mediacloud_client()
favorite_topic_ids = user_db.get_users_lists(user_name(), 'favoriteTopics')
favorited_topics = [user_mc.topic(tid) for tid in favorite_topic_ids]
for t in favorited_topics:
t['isFavorite'] = True
# t['detailInfo'] = get_topic_info_per_snapshot_timespan(t['topics_id'])
return jsonify({'topics': favorited_topics})
def topic_set_favorited(topics_id):
favorite = int(request.form["favorite"])
username = user_name()
if favorite == 1:
user_db.add_item_to_users_list(username, 'favoriteTopics', int(topics_id))
else:
user_db.remove_item_from_users_list(username, 'favoriteTopics', int(topics_id))
return jsonify({'isFavorite': favorite == 1})
def topic_set_favorited(topics_id):
favorite = int(request.form["favorite"])
username = user_name()
if favorite == 1:
user_db.add_item_to_users_list(username, 'favoriteTopics', int(topics_id))
else:
user_db.remove_item_from_users_list(username, 'favoriteTopics', int(topics_id))
return jsonify({'isFavorite': favorite == 1})
def download_favorite_sources():
user_mc = user_admin_mediacloud_client()
user_favorited = db.get_users_lists(user_name(), 'favoriteSources')
favorited_s = [user_mc.media(media_id) for media_id in user_favorited]
for s in favorited_s:
s['isFavorite'] = True
filename = "FavoriteSources.csv"
return download_sources_csv(favorited_s, filename)
def source_set_favorited(media_id):
favorite = request.form["favorite"]
username = user_name()
if int(favorite) == 1:
db.add_item_to_users_list(username, 'favoriteSources', int(media_id))
else:
db.remove_item_from_users_list(username, 'favoriteSources',
int(media_id))
return jsonify({'isFavorite': favorite})
def source_suggest():
user_mc = user_admin_mediacloud_client()
url = request.form['url']
feed_url = request.form['feedurl'] if 'feedurl' in request.form else None
name = request.form['name'] if 'name' in request.form else None
reason = request.form['reason'] if 'reason' in request.form else None
tag_ids_to_add = tag_ids_from_collections_param()
new_suggestion = user_mc.mediaSuggest(url=url, name=name, feed_url=feed_url, reason=reason,
tags_ids=tag_ids_to_add)
# send an email confirmation
email_title = "Thanks for Suggesting " + url
send_html_email(email_title,
[user_name(), '*****@*****.**'],
render_template("emails/source_suggestion_ack.txt", username=user_name(), name=name, url=url,
feed_url=feed_url, reason=reason),
render_template("emails/source_suggestion_ack.html", username=user_name(), name=name, url=url,
feed_url=feed_url, reason=reason)
)
# and return that it worked
return jsonify(new_suggestion)
def save_user_search():
username = user_name()
queries = json.loads(request.form['queries'])
name = request.form['queryName']
timestamp = request.form['timestamp']
user_db.add_item_to_users_list(username, 'searches', {
'queries': queries,
'queryName': name,
'timestamp': timestamp,
})
return jsonify({'savedQuery': request.form['queryName']})
def _email_batch_source_update_results(audit_feedback):
email_title = "Source Batch Updates"
content_title = "You just uploaded {} sources to a collection.".format(len(audit_feedback))
updated_sources = []
for updated in audit_feedback:
updated_sources.append(updated)
content_body = updated_sources
action_text = "Login to Media Cloud"
action_url = "https://sources.mediacloud.org/#/login"
# send an email confirmation
send_html_email(email_title,
[user_name(), '*****@*****.**'],
render_template("emails/source_batch_upload_ack.txt",
content_title=content_title, content_body=content_body, action_text=action_text,
action_url=action_url),
render_template("emails/source_batch_upload_ack.html",
email_title=email_title, content_title=content_title, content_body=content_body,
action_text=action_text, action_url=action_url)
)
def source_stats(media_id):
username = user_name()
user_mc = user_admin_mediacloud_client()
results = {}
# story count
media_query = "(media_id:{})".format(media_id)
source_specific_story_count = cached_source_story_count(
username, media_query)
results['story_count'] = source_specific_story_count
# health
media_health = _cached_media_source_health(username, media_id)
results['num_stories_90'] = media_health[
'num_stories_90'] if 'num_stories_90' in media_health else None
results['start_date'] = media_health[
'start_date'] if 'start_date' in media_health else None
info = _media_source_details(media_id)
user_can_see_private_collections = user_has_auth_role(ROLE_MEDIA_EDIT)
visible_collections = [
c for c in info['media_source_tags']
if ((c['tag_sets_id'] in VALID_COLLECTION_TAG_SETS_IDS) and (
(c['show_on_media'] == 1) or user_can_see_private_collections))
]
results['collection_count'] = len(visible_collections)
# geography tags
tag_specific_story_count = user_mc.storyTagCount(
solr_query=media_query, tag_sets_id=TAG_SET_GEOCODER_VERSION)
ratio_geo_tagged_count = float(
tag_specific_story_count[0]['count']) / float(
source_specific_story_count) if len(
tag_specific_story_count) > 0 else 0
results['geoPct'] = ratio_geo_tagged_count
# nyt theme
tag_specific_story_count = user_mc.storyTagCount(
solr_query=media_query, tag_sets_id=TAG_SET_NYT_LABELS_VERSION)
ratio_nyt_tagged_count = float(
tag_specific_story_count[0]['count']) / float(
source_specific_story_count) if len(
tag_specific_story_count) > 0 else 0
results['nytPct'] = ratio_nyt_tagged_count
return jsonify(results)
def source_stats(media_id):
username = user_name()
user_mc = user_admin_mediacloud_client()
results = {}
# story count
media_query = "(media_id:{})".format(media_id)
source_specific_story_count = apicache.source_story_count(user_mediacloud_key(), media_query)
results['story_count'] = source_specific_story_count
# health
media_health = _cached_media_source_health(username, media_id)
results['num_stories_90'] = media_health['num_stories_90'] if 'num_stories_90' in media_health else None
results['start_date'] = media_health['start_date'] if 'start_date' in media_health else None
info = _media_source_details(media_id)
user_can_see_private_collections = user_has_auth_role(ROLE_MEDIA_EDIT)
visible_collections = [c for c in info['media_source_tags']
if ((c['tag_sets_id'] in VALID_COLLECTION_TAG_SETS_IDS) and
((c['show_on_media'] == 1) or user_can_see_private_collections))]
results['collection_count'] = len(visible_collections)
# geography tags
results['geoPct'] = apicache.tag_coverage_pct(user_mediacloud_key(), media_query, TAG_SET_GEOCODER_VERSION)
# nyt theme
results['nytPct'] = apicache.tag_coverage_pct(user_mediacloud_key(), media_query, TAG_SET_NYT_LABELS_VERSION)
return jsonify(results)
def notebook_add():
content = json.loads(request.form['content'])
object_id = db.save_notebook_entry(auth.user_name(), content)
return jsonify({'id': str(object_id)})
def add_user_favorite_flag_to_sources(sources):
user_favorited = db.get_users_lists(user_name(), 'favoriteSources')
for s in sources:
s['isFavorite'] = int(s['media_id']) in user_favorited
return sources
def delete_user_search():
username = user_name()
result = user_db.remove_item_from_users_list(
username, 'searches', {'timestamp': int(request.form['timestamp'])})
return jsonify({'success': result.raw_result})
def load_user_searches():
username = user_name()
search_list = db.get_users_lists(username, 'searches')
return jsonify({'list': search_list})
def add_user_favorite_flag_to_collections(collections):
user_favorited = db.get_users_lists(user_name(), 'favoriteCollections')
for c in collections:
c['isFavorite'] = int(c['tags_id']) in user_favorited
return collections
def save_user_search():
username = user_name()
db.add_item_to_users_list(username, 'searches', request.args)
return jsonify({'savedQuery': request.args['queryName']})
def delete_user_search():
username = user_name()
result = db.remove_item_from_users_list(username, 'searches', request.args)
return jsonify({'success': result.raw_result})
def _add_user_favorite_flag_to_topics(topics):
user_favorited = user_db.get_users_lists(user_name(), 'favoriteTopics')
for t in topics:
t['isFavorite'] = t['topics_id'] in user_favorited
return topics
def add_user_favorite_flag_to_topics(topics):
user_favorited = user_db.get_users_lists(user_name(), 'favoriteTopics')
for t in topics:
t['isFavorite'] = t['topics_id'] in user_favorited
return topics
def save_user_query():
username = user_name()
# TODO any checking here?
db.add_item_to_users_list(username, 'queries', request.args)
return jsonify({'savedQuery': request.args['query_string']})