def topic_stories(topics_id):
if access_public_topic(topics_id):
stories = topic_story_list(TOOL_API_KEY, topics_id, snapshots_id=None, timespans_id=None, foci_id=None, q=None)
elif is_user_logged_in():
stories = topic_story_list(user_mediacloud_key(), topics_id)
else:
return jsonify({'status': 'Error', 'message': 'Invalid attempt'})
return jsonify(stories)
def topic_stories(topics_id):
if access_public_topic(topics_id):
stories = apicache.topic_story_list(TOOL_API_KEY, topics_id, snapshots_id=None, timespans_id=None,
foci_id=None, q=None)
elif is_user_logged_in():
stories = apicache.topic_story_list(user_mediacloud_key(), topics_id)
else:
return jsonify({'status': 'Error', 'message': 'Invalid attempt'})
return jsonify(stories)
def story(topics_id, stories_id):
story_topic_info = apicache.topic_story_list(user_mediacloud_key(),
topics_id,
stories_id=stories_id)
story_topic_info = story_topic_info['stories'][0]
try:
story_info = base_apicache.story(
stories_id) # add in other fields from regular call
for k in story_info.keys():
story_topic_info[k] = story_info[k]
for tag in story_info['story_tags']:
if tag['tag_sets_id'] == tag_util.GEO_TAG_SET:
geonames_id = int(tag['tag'][9:])
try:
tag['geoname'] = _cached_geoname(geonames_id,
story['language'])
except Exception as e:
# query to CLIFF failed :-( handle it gracefully
logger.exception(e)
tag['geoname'] = {}
except MCException:
logger.warning(
"Story {} wasn't found in a regular story API call, but is it topic {}"
.format(stories_id, topics_id))
return jsonify(story_topic_info)
def media_inlinks(topics_id, media_id):
user_mc = user_admin_mediacloud_client()
sort = validated_sort(request.args.get('sort'))
limit = request.args.get('limit')
inlinks = topic_story_list(user_mediacloud_key(), topics_id,
link_to_media_id=media_id, sort=sort, limit=limit)
return jsonify(inlinks)
def story(topics_id, stories_id):
local_mc = None
if access_public_topic(topics_id):
local_mc = mc
story_topic_info = topic_story_list(TOOL_API_KEY, topics_id, stories_id=stories_id)['stories'][0]
elif is_user_logged_in():
local_mc = user_mediacloud_client()
story_topic_info = topic_story_list(user_mediacloud_key(), topics_id, stories_id=stories_id)['stories'][0]
'''
all_fb_count = []
more_fb_count = True
link_id = 0
while more_fb_count:
fb_page = local_mc.topicStoryListFacebookData(topics_id, limit=100, link_id=link_id)
all_fb_count = all_fb_count + fb_page['counts']
if 'next' in fb_page['link_ids']:
link_id = fb_page['link_ids']['next']
more_fb_count = True
else:
more_fb_count = False
for fb_item in all_fb_count:
if int(fb_item['stories_id']) == int(stories_id):
story_topic_info['facebook_collection_date'] = fb_item['facebook_api_collect_date']
'''
else:
return jsonify({'status':'Error', 'message': 'Invalid attempt'})
story_info = local_mc.story(stories_id) # add in other fields from regular call
for k in story_info.keys():
if k not in story_topic_info.keys():
story_topic_info[k] = story_info[k]
for tag in story_info['story_tags']:
if tag['tag_sets_id'] == tag_util.GEO_TAG_SET:
geonames_id = int(tag['tag'][9:])
try:
tag['geoname'] = _cached_geoname(geonames_id)
except Exception as e:
# query to CLIFF failed :-( handle it gracefully
logger.exception(e)
tag['geoname'] = {}
return jsonify(story_topic_info)
def media_stories(topics_id, media_id):
sort = validated_sort(request.args.get('sort'))
limit = request.args.get('limit')
stories = apicache.topic_story_list(user_mediacloud_key(),
topics_id,
media_id=media_id,
sort=sort,
limit=limit)
return jsonify(stories)
def all_media_outlinks(topics_id, media_id):
all_stories = []
more_stories = True
link_id = 0
sort = validated_sort(request.args.get('sort'))
while more_stories:
page = apicache.topic_story_list(user_mediacloud_key(), topics_id, link_from_media_id=media_id,
sort=sort, link_id=link_id, limit=1000)
story_list = page['stories']
all_stories = all_stories + story_list
if 'next' in page['link_ids']:
link_id = page['link_ids']['next']
more_stories = True
else:
more_stories = False
return jsonify({"stories": all_stories})
def story(topics_id, stories_id):
if is_user_logged_in():
local_mc = user_mediacloud_client()
story_topic_info = apicache.topic_story_list(user_mediacloud_key(), topics_id, stories_id=stories_id)
story_topic_info = story_topic_info['stories'][0]
'''
all_fb_count = []
more_fb_count = True
link_id = 0
while more_fb_count:
fb_page = local_mc.topicStoryListFacebookData(topics_id, limit=100, link_id=link_id)
all_fb_count = all_fb_count + fb_page['counts']
if 'next' in fb_page['link_ids']:
link_id = fb_page['link_ids']['next']
more_fb_count = True
else:
more_fb_count = False
for fb_item in all_fb_count:
if int(fb_item['stories_id']) == int(stories_id):
story_topic_info['facebook_collection_date'] = fb_item['facebook_api_collect_date']
'''
else:
return jsonify({'status': 'Error', 'message': 'Invalid attempt'})
try:
story_info = local_mc.story(stories_id) # add in other fields from regular call
for k in story_info.keys():
story_topic_info[k] = story_info[k]
for tag in story_info['story_tags']:
if tag['tag_sets_id'] == tag_util.GEO_TAG_SET:
geonames_id = int(tag['tag'][9:])
try:
tag['geoname'] = _cached_geoname(geonames_id)
except Exception as e:
# query to CLIFF failed :-( handle it gracefully
logger.exception(e)
tag['geoname'] = {}
except MCException:
logger.warning("Story {} wasn't found in a regular story API call, but is it topic {}".format(
stories_id, topics_id
))
return jsonify(story_topic_info)
def all_media_outlinks(topics_id, media_id):
all_stories = []
more_stories = True
link_id = 0
sort = validated_sort(request.args.get('sort'))
while more_stories:
page = apicache.topic_story_list(user_mediacloud_key(),
topics_id,
link_from_media_id=media_id,
sort=sort,
link_id=link_id,
limit=1000)
story_list = page['stories']
all_stories = all_stories + story_list
if 'next' in page['link_ids']:
link_id = page['link_ids']['next']
more_stories = True
else:
more_stories = False
return jsonify({"stories": all_stories})
def story_inlinks(topics_id, stories_id):
inlinks = apicache.topic_story_list(TOOL_API_KEY,
topics_id,
link_to_stories_id=stories_id,
limit=50)
return jsonify(inlinks)
def story_outlinks(topics_id, stories_id):
outlinks = apicache.topic_story_list(user_mediacloud_key(), topics_id, link_from_stories_id=stories_id, limit=50)
return jsonify(outlinks)
def story_inlinks(topics_id, stories_id):
inlinks = apicache.topic_story_list(user_mediacloud_key(), topics_id,
link_to_stories_id=stories_id, limit=50)
return jsonify(inlinks)
def topic_word_stories(topics_id, word):
response = apicache.topic_story_list(user_mediacloud_key(), topics_id, q=word)
return jsonify(response)
def topic_provider_stories(topics_id):
optional_args = _parse_stories_optional_arguments()
results = apicache.topic_story_list(user_mediacloud_key(), topics_id,
**optional_args)
return jsonify(results)
def topic_word_stories(topics_id, word):
response = topic_story_list(user_mediacloud_key(), topics_id, q=word)
return jsonify(response)
def media_inlinks(topics_id, media_id):
sort = validated_sort(request.args.get('sort'))
limit = request.args.get('limit')
inlinks = apicache.topic_story_list(user_mediacloud_key(), topics_id,
link_to_media_id=media_id, sort=sort, limit=limit)
return jsonify(inlinks)
def stream_story_list_csv(user_mc_key, filename, topics_id, **kwargs):
'''
Helper method to stream a list of stories back to the client as a csv. Any args you pass in will be
simply be passed on to a call to topicStoryList.
'''
as_attachment = kwargs['as_attachment'] if 'as_attachment' in kwargs else True
fb_data = kwargs['fb_data'] if 'fb_data' in kwargs else False
all_stories = []
more_stories = True
params = kwargs
if 'as_attachment' in params:
del params['as_attachment']
if 'fb_data' in params:
del params['fb_data']
params['limit'] = 1000 # an arbitrary value to let us page through with big pages
props = ['stories_id', 'publish_date', 'date_is_reliable',
'title', 'url', 'media_id', 'media_name',
'media_inlink_count', 'inlink_count', 'outlink_count', 'bitly_click_count',
'facebook_share_count', 'language', 'subtopics', 'themes']
user_mc = user_mediacloud_client()
try:
while more_stories:
page = topic_story_list(user_mc_key, topics_id, **params)
# need to make another call to fetch the tags :-(
story_ids = [str(s['stories_id']) for s in page['stories']]
stories_with_tags = user_mc.storyList('stories_id:('+" ".join(story_ids)+")", rows=kwargs['limit'])
story_ids_to_tags = {int(s['stories_id']): s['story_tags'] for s in stories_with_tags}
for s in page['stories']:
s['themes'] = '?' # means we haven't processed it for themes yet
s['subtopics'] = '?' # fill it in for safety
stories_id = s['stories_id']
if stories_id in story_ids_to_tags:
story_tags = story_ids_to_tags[stories_id]
story_tag_ids = [t['tags_id'] for t in story_tags]
# add in the names of any themes
if tag_util.NYT_LABELER_1_0_0_TAG_ID in story_tag_ids:
s['themes'] = ",".join([t['tag'] for t in story_tags
if t['tag_sets_id'] == tag_util.NYT_LABELS_TAG_SET_ID])
# not doing geonames places here because it would take too long to fetch the name with `_cached_geonames`
# add in the names of any subtopics
foci_names = [f['name'] for f in s['foci']]
s['subtopics'] = ",".join(foci_names)
all_stories = all_stories + page['stories']
if 'next' in page['link_ids']:
params['link_id'] = page['link_ids']['next']
more_stories = True
else:
more_stories = False
if fb_data:
all_fb_count = []
more_fb_count = True
link_id = 0
local_mc = user_admin_mediacloud_client()
while more_fb_count:
fb_page = local_mc.topicStoryListFacebookData(topics_id, limit=100, link_id=link_id)
all_fb_count = all_fb_count + fb_page['counts']
if 'next' in fb_page['link_ids']:
link_id = fb_page['link_ids']['next']
more_fb_count = True
else:
more_fb_count = False
# now iterate through each list and set up the fb collection date
for story in all_stories:
for fb_item in all_fb_count:
if int(fb_item['stories_id']) == int(story['stories_id']):
story['facebook_collection_date'] = fb_item['facebook_api_collect_date']
props.append('facebook_collection_date')
return csv.stream_response(all_stories, props, filename, as_attachment=as_attachment)
except Exception as e:
logger.exception(e)
return json.dumps({'error': str(e)}, separators=(',', ':')), 400
def story_inlinks(topics_id, stories_id):
inlinks = topic_story_list(user_mediacloud_key(), topics_id, link_to_stories_id=stories_id, limit=50)
return jsonify(inlinks)
def story_outlinks(topics_id, stories_id):
outlinks = topic_story_list(user_mediacloud_key(), topics_id, link_from_stories_id=stories_id, limit=50)
return jsonify(outlinks)
