def topic_update(topics_id):
user_mc = user_admin_mediacloud_client()
# top five cannot be empty fyi
args = {
'name': request.form['name'] if 'name' in request.form else None,
'description': request.form['description'] if 'description' in request.form else None,
'solr_seed_query': request.form['solr_seed_query'] if 'solr_seed_query' in request.form else None,
'start_date': request.form['start_date'] if 'start_date' in request.form else None,
'end_date': request.form['end_date'] if 'end_date' in request.form else None,
'is_public': request.form['is_public'] if 'is_public' in request.form else None,
'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None,
'ch_monitor_id': request.form['ch_monitor_id'] if 'ch_monitor_id' in request.form
and request.form['ch_monitor_id'] != 'null'
and len(request.form['ch_monitor_id']) > 0 else None,
'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None,
'max_stories': request.form['max_stories'] if 'max_stories' in request.form else None,
'twitter_topics_id': request.form['twitter_topics_id'] if 'twitter_topics_id' in request.form else None
}
# parse out any sources and collections to add
media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]'] if 'sources[]' in request.form else '')
tag_ids_to_add = ids_from_comma_separated_str(request.form['collections[]']
if 'collections[]' in request.form else '')
# hack to support twitter-only topics
if (len(media_ids_to_add) is 0) and (len(tag_ids_to_add) is 0):
media_ids_to_add = None
tag_ids_to_add = None
result = user_mc.topicUpdate(topics_id, media_ids=media_ids_to_add, media_tags_ids=tag_ids_to_add, **args)
return topic_summary(result['topics'][0]['topics_id']) # give them back new data, so they can update the client
def topic_create():
user_mc = user_mediacloud_client()
name = request.form['name']
description = request.form['description']
solr_seed_query = request.form['solr_seed_query']
start_date = request.form['start_date']
end_date = request.form['end_date']
optional_args = {
'is_public':
request.form['is_public'] if 'is_public' in request.form else None,
'is_logogram':
request.form['is_logogram'] if 'is_logogram' in request.form else None,
'ch_monitor_id':
request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0
and request.form['ch_monitor_id'] != 'null' else None,
'max_iterations':
request.form['max_iterations']
if 'max_iterations' in request.form else None,
'max_stories':
request.form['max_stories'] if 'max_stories' in request.form
and request.form['max_stories'] != 'null' else
flask_login.current_user.profile['max_topic_stories'],
}
# parse out any sources and collections to add
media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]'])
tag_ids_to_add = ids_from_comma_separated_str(
request.form['collections[]'])
try:
topic_result = user_mc.topicCreate(name=name,
description=description,
solr_seed_query=solr_seed_query,
start_date=start_date,
end_date=end_date,
media_ids=media_ids_to_add,
media_tags_ids=tag_ids_to_add,
**optional_args)['topics'][0]
topics_id = topic_result['topics_id']
logger.info("Created new topic \"{}\" as {}".format(name, topics_id))
# if this includes any of the US-centric collections, add the retweet partisanship subtopic by default
if set(tag_ids_to_add).intersection(US_COLLECTIONS):
add_retweet_partisanship_to_topic(
topic_result['topics_id'], 'Retweet Partisanship',
'Subtopics driven by our analysis of Twitter followers of Trump and Clinton during the 2016 election season. Each media soure is scored based on the ratio of retweets of their stories in those two groups.'
)
# client will either make a empty snapshot, or a spidering one
return topic_summary(topics_id)
except Exception as e:
logging.error("Topic creation failed {}".format(name))
logging.exception(e)
return json_error_response(str(e), 500)
except mediacloud.error.MCException as e:
logging.error("Topic creation failed {}".format(name))
logging.exception(e)
return json_error_response(e.message, e.status_code)
def api_topics_preview_word_count():
user_mc = user_admin_mediacloud_client()
solr_query = concatenate_query_for_solr(solr_seed_query=request.form['q'],
media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None,
tags_ids=ids_from_comma_separated_str(request.form['collections[]'])) if 'collections[]' in request.form else None,
fq = concatenate_solr_dates(start_date=request.form['start_date'],
end_date=request.form['end_date'])
word_count_result = user_mc.wordCount(solr_query=solr_query, solr_filter=fq)
return jsonify(word_count_result) # give them back new data, so they can update the client
def api_topics_preview_story_sample():
user_mc = user_mediacloud_client()
solr_query = concatenate_query_for_solr(solr_seed_query=request.form['q'],
media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None,
tags_ids=ids_from_comma_separated_str(request.form['collections[]'])) if 'collections[]' in request.form else None,
fq = concatenate_solr_dates(start_date=request.form['start_date'],
end_date=request.form['end_date'])
num_stories = request.form['rows']
story_count_result = user_mc.storyList(solr_query=solr_query, solr_filter=fq, sort=user_mc.SORT_RANDOM, rows=num_stories)
return jsonify(story_count_result)
def topic_create():
user_mc = user_admin_mediacloud_client()
name = request.form['name']
description = request.form['description']
solr_seed_query = request.form['solr_seed_query']
start_date = request.form['start_date']
end_date = request.form['end_date']
optional_args = {
'is_public':
request.form['is_public'] if 'is_public' in request.form else None,
'is_logogram':
request.form['is_logogram'] if 'is_logogram' in request.form else None,
'ch_monitor_id':
request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0
and request.form['ch_monitor_id'] != 'null' else None,
'max_iterations':
request.form['max_iterations']
if 'max_iterations' in request.form else None,
'max_stories':
request.form['max_stories'] if 'max_stories' in request.form
and request.form['max_stories'] != 'null' else None,
}
# parse out any sources and collections to add
media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]'])
tag_ids_to_add = ids_from_comma_separated_str(
request.form['collections[]'])
try:
topic_result = user_mc.topicCreate(name=name,
description=description,
solr_seed_query=solr_seed_query,
start_date=start_date,
end_date=end_date,
media_ids=media_ids_to_add,
media_tags_ids=tag_ids_to_add,
**optional_args)['topics'][0]
topic_id = topic_result['topics_id']
logger.info("Created new topic \"{}\" as {}".format(name, topic_id))
spider_job = user_mc.topicSpider(
topic_id) # kick off a spider, which will also generate a snapshot
logger.info(" spider result = {}".format(json.dumps(spider_job)))
results = user_mc.topic(topic_id)
results['spider_job_state'] = spider_job
return jsonify(
results) # give them back new data, so they can update the client
except Exception as e:
logging.error("Topic creation failed {}".format(name))
logging.exception(e)
return json_error_response(e.message, e.status_code)
def topic_update(topics_id):
# update the seed query first 5 MUST be filled in)
args = {
'name':
request.form['name'] if 'name' in request.form else None,
'description':
request.form['description'] if 'description' in request.form else None,
'solr_seed_query':
request.form['solr_seed_query']
if 'solr_seed_query' in request.form else None,
'start_date':
request.form['start_date'] if 'start_date' in request.form else None,
'end_date':
request.form['end_date'] if 'end_date' in request.form else None,
'is_public':
request.form['is_public'] if 'is_public' in request.form else None,
'is_logogram':
request.form['is_logogram'] if 'is_logogram' in request.form else None,
'ch_monitor_id':
request.form['ch_monitor_id'] if 'ch_monitor_id' in request.form
and request.form['ch_monitor_id'] is not None
and request.form['ch_monitor_id'] != 'null'
and len(request.form['ch_monitor_id']) > 0 else None,
'max_iterations':
request.form['max_iterations']
if 'max_iterations' in request.form else None,
'max_stories':
request.form['max_stories'] if 'max_stories' in request.form
and request.form['max_stories'] != 'null' else
current_user.profile['max_topic_stories'],
'twitter_topics_id':
request.form['twitter_topics_id']
if 'twitter_topics_id' in request.form else None
}
# parse out any sources and collections to add
media_ids_to_add = ids_from_comma_separated_str(
request.form['sources[]'] if 'sources[]' in request.form else '')
tag_ids_to_add = ids_from_comma_separated_str(
request.form['collections[]'] if 'collections[]' in
request.form else '')
# hack to support twitter-only topics
if (len(media_ids_to_add) is 0) and (len(tag_ids_to_add) is 0):
media_ids_to_add = None
tag_ids_to_add = None
# update the seed query (the client will start the spider themselves
user_mc = user_mediacloud_client()
result = user_mc.topicUpdate(topics_id,
media_ids=media_ids_to_add,
media_tags_ids=tag_ids_to_add,
**args)
return topic_summary(
topics_id) # give them back new data, so they can update the client
def api_topics_preview_split_story_count():
user_mc = user_admin_mediacloud_client()
solr_query = concatenate_query_for_solr(solr_seed_query=request.form['q'],
media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None,
tags_ids=ids_from_comma_separated_str(request.form['collections[]'])) if 'collections[]' in request.form else None,
fq = concatenate_solr_dates(start_date=request.form['start_date'],
end_date=request.form['end_date'])
results = user_mc.storyCount(solr_query=solr_query, solr_filter=fq, split=True)
total_stories = 0
for c in results['counts']:
total_stories += c['count']
results['total_story_count'] = total_stories
return jsonify({'results': results})
def topic_update(topics_id):
user_mc = user_admin_mediacloud_client()
# top five cannot be empty fyi
args = {
'name':
request.form['name'],
'description':
request.form['description'],
'solr_seed_query':
request.form['solr_seed_query'],
'start_date':
request.form['start_date'],
'end_date':
request.form['end_date'],
'is_public':
request.form['is_public'] if 'is_public' in request.form else None,
'is_logogram':
request.form['is_logogram'] if 'is_logogram' in request.form else None,
'ch_monitor_id':
request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0
and request.form['ch_monitor_id'] != 'null' else None,
'max_iterations':
request.form['max_iterations']
if 'max_iterations' in request.form else None,
'max_stories':
request.form['max_stories'] if 'max_stories' in request.form else None,
'twitter_topics_id':
request.form['twitter_topics_id']
if 'twitter_topics_id' in request.form else None
}
# parse out any sources and collections to add
media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]'])
tag_ids_to_add = ids_from_comma_separated_str(
request.form['collections[]'])
# hack to support twitter-only topics
if (len(media_ids_to_add) is 0) and (len(tag_ids_to_add) is 0):
media_ids_to_add = None
tag_ids_to_add = None
result = user_mc.topicUpdate(topics_id,
media_ids=media_ids_to_add,
media_tags_ids=tag_ids_to_add,
**args)
return topic_summary(
result['topics'][0]['topics_id']
) # give them back new data, so they can update the client
def api_topics_preview_word_count():
user_mc = user_admin_mediacloud_client()
solr_query = concatenate_query_for_solr(
solr_seed_query=request.form['q'],
media_ids=ids_from_comma_separated_str(request.form['sources[]'])
if 'sources[]' in request.form else None,
tags_ids=ids_from_comma_separated_str(request.form['collections[]'])
) if 'collections[]' in request.form else None,
fq = concatenate_solr_dates(start_date=request.form['start_date'],
end_date=request.form['end_date'])
word_count_result = user_mc.wordCount(solr_query=solr_query,
solr_filter=fq)
return jsonify(word_count_result
) # give them back new data, so they can update the client
def api_topics_preview_story_sample():
user_mc = user_mediacloud_client()
solr_query = concatenate_query_for_solr(
solr_seed_query=request.form['q'],
media_ids=ids_from_comma_separated_str(request.form['sources[]'])
if 'sources[]' in request.form else None,
tags_ids=ids_from_comma_separated_str(request.form['collections[]'])
) if 'collections[]' in request.form else None,
fq = concatenate_solr_dates(start_date=request.form['start_date'],
end_date=request.form['end_date'])
story_count_result = user_mc.storyList(solr_query=solr_query,
solr_filter=fq,
sort=user_mc.SORT_RANDOM)
return jsonify(story_count_result)
def api_topics_preview_split_story_count():
user_mc = user_admin_mediacloud_client()
solr_query = concatenate_query_for_solr(
solr_seed_query=request.form['q'],
media_ids=ids_from_comma_separated_str(request.form['sources[]'])
if 'sources[]' in request.form else None,
tags_ids=ids_from_comma_separated_str(request.form['collections[]'])
) if 'collections[]' in request.form else None,
fq = concatenate_solr_dates(start_date=request.form['start_date'],
end_date=request.form['end_date'])
results = user_mc.storyCount(solr_query=solr_query,
solr_filter=fq,
split=True)
total_stories = 0
for c in results['counts']:
total_stories += c['count']
results['total_story_count'] = total_stories
return jsonify({'results': results})
def topic_create():
user_mc = user_admin_mediacloud_client()
name = request.form['name']
description = request.form['description']
solr_seed_query = request.form['solr_seed_query']
start_date = request.form['start_date']
end_date = request.form['end_date']
optional_args = {
'is_public': request.form['is_public'] if 'is_public' in request.form else None,
'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None,
'ch_monitor_id': request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0 and request.form['ch_monitor_id'] != 'null' else None,
'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None,
'max_stories': request.form['max_stories'] if 'max_stories' in request.form and request.form['max_stories'] != 'null' else None,
}
# parse out any sources and collections to add
media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]'])
tag_ids_to_add = ids_from_comma_separated_str(request.form['collections[]'])
try:
topic_result = user_mc.topicCreate(name=name, description=description, solr_seed_query=solr_seed_query,
start_date=start_date, end_date=end_date, media_ids=media_ids_to_add,
media_tags_ids=tag_ids_to_add, **optional_args)['topics'][0]
topic_id = topic_result['topics_id']
logger.info("Created new topic \"{}\" as {}".format(name, topic_id))
if set(tag_ids_to_add).intersection(US_COLLECTIONS):
create_retweet_partisanship_focal_set(topic_result['topics_id'])
spider_job = user_mc.topicSpider(topic_id) # kick off a spider, which will also generate a snapshot
logger.info(" spider result = {}".format(json.dumps(spider_job)))
results = user_mc.topic(topic_id)
results['spider_job_state'] = spider_job
return jsonify(results) # give them back new data, so they can update the client
except Exception as e:
logging.error("Topic creation failed {}".format(name))
logging.exception(e)
return json_error_response(e.message, e.status_code)
