def topic_update(topics_id): user_mc = user_admin_mediacloud_client() # top five cannot be empty fyi args = { 'name': request.form['name'] if 'name' in request.form else None, 'description': request.form['description'] if 'description' in request.form else None, 'solr_seed_query': request.form['solr_seed_query'] if 'solr_seed_query' in request.form else None, 'start_date': request.form['start_date'] if 'start_date' in request.form else None, 'end_date': request.form['end_date'] if 'end_date' in request.form else None, 'is_public': request.form['is_public'] if 'is_public' in request.form else None, 'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None, 'ch_monitor_id': request.form['ch_monitor_id'] if 'ch_monitor_id' in request.form and request.form['ch_monitor_id'] != 'null' and len(request.form['ch_monitor_id']) > 0 else None, 'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None, 'max_stories': request.form['max_stories'] if 'max_stories' in request.form else None, 'twitter_topics_id': request.form['twitter_topics_id'] if 'twitter_topics_id' in request.form else None } # parse out any sources and collections to add media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]'] if 'sources[]' in request.form else '') tag_ids_to_add = ids_from_comma_separated_str(request.form['collections[]'] if 'collections[]' in request.form else '') # hack to support twitter-only topics if (len(media_ids_to_add) is 0) and (len(tag_ids_to_add) is 0): media_ids_to_add = None tag_ids_to_add = None result = user_mc.topicUpdate(topics_id, media_ids=media_ids_to_add, media_tags_ids=tag_ids_to_add, **args) return topic_summary(result['topics'][0]['topics_id']) # give them back new data, so they can update the client
def topic_create(): user_mc = user_mediacloud_client() name = request.form['name'] description = request.form['description'] solr_seed_query = request.form['solr_seed_query'] start_date = request.form['start_date'] end_date = request.form['end_date'] optional_args = { 'is_public': request.form['is_public'] if 'is_public' in request.form else None, 'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None, 'ch_monitor_id': request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0 and request.form['ch_monitor_id'] != 'null' else None, 'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None, 'max_stories': request.form['max_stories'] if 'max_stories' in request.form and request.form['max_stories'] != 'null' else flask_login.current_user.profile['max_topic_stories'], } # parse out any sources and collections to add media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]']) tag_ids_to_add = ids_from_comma_separated_str( request.form['collections[]']) try: topic_result = user_mc.topicCreate(name=name, description=description, solr_seed_query=solr_seed_query, start_date=start_date, end_date=end_date, media_ids=media_ids_to_add, media_tags_ids=tag_ids_to_add, **optional_args)['topics'][0] topics_id = topic_result['topics_id'] logger.info("Created new topic \"{}\" as {}".format(name, topics_id)) # if this includes any of the US-centric collections, add the retweet partisanship subtopic by default if set(tag_ids_to_add).intersection(US_COLLECTIONS): add_retweet_partisanship_to_topic( topic_result['topics_id'], 'Retweet Partisanship', 'Subtopics driven by our analysis of Twitter followers of Trump and Clinton during the 2016 election season. Each media soure is scored based on the ratio of retweets of their stories in those two groups.' ) # client will either make a empty snapshot, or a spidering one return topic_summary(topics_id) except Exception as e: logging.error("Topic creation failed {}".format(name)) logging.exception(e) return json_error_response(str(e), 500) except mediacloud.error.MCException as e: logging.error("Topic creation failed {}".format(name)) logging.exception(e) return json_error_response(e.message, e.status_code)
def api_topics_preview_word_count(): user_mc = user_admin_mediacloud_client() solr_query = concatenate_query_for_solr(solr_seed_query=request.form['q'], media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None, tags_ids=ids_from_comma_separated_str(request.form['collections[]'])) if 'collections[]' in request.form else None, fq = concatenate_solr_dates(start_date=request.form['start_date'], end_date=request.form['end_date']) word_count_result = user_mc.wordCount(solr_query=solr_query, solr_filter=fq) return jsonify(word_count_result) # give them back new data, so they can update the client
def api_topics_preview_story_sample(): user_mc = user_mediacloud_client() solr_query = concatenate_query_for_solr(solr_seed_query=request.form['q'], media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None, tags_ids=ids_from_comma_separated_str(request.form['collections[]'])) if 'collections[]' in request.form else None, fq = concatenate_solr_dates(start_date=request.form['start_date'], end_date=request.form['end_date']) num_stories = request.form['rows'] story_count_result = user_mc.storyList(solr_query=solr_query, solr_filter=fq, sort=user_mc.SORT_RANDOM, rows=num_stories) return jsonify(story_count_result)
def topic_create(): user_mc = user_admin_mediacloud_client() name = request.form['name'] description = request.form['description'] solr_seed_query = request.form['solr_seed_query'] start_date = request.form['start_date'] end_date = request.form['end_date'] optional_args = { 'is_public': request.form['is_public'] if 'is_public' in request.form else None, 'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None, 'ch_monitor_id': request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0 and request.form['ch_monitor_id'] != 'null' else None, 'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None, 'max_stories': request.form['max_stories'] if 'max_stories' in request.form and request.form['max_stories'] != 'null' else None, } # parse out any sources and collections to add media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]']) tag_ids_to_add = ids_from_comma_separated_str( request.form['collections[]']) try: topic_result = user_mc.topicCreate(name=name, description=description, solr_seed_query=solr_seed_query, start_date=start_date, end_date=end_date, media_ids=media_ids_to_add, media_tags_ids=tag_ids_to_add, **optional_args)['topics'][0] topic_id = topic_result['topics_id'] logger.info("Created new topic \"{}\" as {}".format(name, topic_id)) spider_job = user_mc.topicSpider( topic_id) # kick off a spider, which will also generate a snapshot logger.info(" spider result = {}".format(json.dumps(spider_job))) results = user_mc.topic(topic_id) results['spider_job_state'] = spider_job return jsonify( results) # give them back new data, so they can update the client except Exception as e: logging.error("Topic creation failed {}".format(name)) logging.exception(e) return json_error_response(e.message, e.status_code)
def topic_update(topics_id): # update the seed query first 5 MUST be filled in) args = { 'name': request.form['name'] if 'name' in request.form else None, 'description': request.form['description'] if 'description' in request.form else None, 'solr_seed_query': request.form['solr_seed_query'] if 'solr_seed_query' in request.form else None, 'start_date': request.form['start_date'] if 'start_date' in request.form else None, 'end_date': request.form['end_date'] if 'end_date' in request.form else None, 'is_public': request.form['is_public'] if 'is_public' in request.form else None, 'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None, 'ch_monitor_id': request.form['ch_monitor_id'] if 'ch_monitor_id' in request.form and request.form['ch_monitor_id'] is not None and request.form['ch_monitor_id'] != 'null' and len(request.form['ch_monitor_id']) > 0 else None, 'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None, 'max_stories': request.form['max_stories'] if 'max_stories' in request.form and request.form['max_stories'] != 'null' else current_user.profile['max_topic_stories'], 'twitter_topics_id': request.form['twitter_topics_id'] if 'twitter_topics_id' in request.form else None } # parse out any sources and collections to add media_ids_to_add = ids_from_comma_separated_str( request.form['sources[]'] if 'sources[]' in request.form else '') tag_ids_to_add = ids_from_comma_separated_str( request.form['collections[]'] if 'collections[]' in request.form else '') # hack to support twitter-only topics if (len(media_ids_to_add) is 0) and (len(tag_ids_to_add) is 0): media_ids_to_add = None tag_ids_to_add = None # update the seed query (the client will start the spider themselves user_mc = user_mediacloud_client() result = user_mc.topicUpdate(topics_id, media_ids=media_ids_to_add, media_tags_ids=tag_ids_to_add, **args) return topic_summary( topics_id) # give them back new data, so they can update the client
def api_topics_preview_split_story_count(): user_mc = user_admin_mediacloud_client() solr_query = concatenate_query_for_solr(solr_seed_query=request.form['q'], media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None, tags_ids=ids_from_comma_separated_str(request.form['collections[]'])) if 'collections[]' in request.form else None, fq = concatenate_solr_dates(start_date=request.form['start_date'], end_date=request.form['end_date']) results = user_mc.storyCount(solr_query=solr_query, solr_filter=fq, split=True) total_stories = 0 for c in results['counts']: total_stories += c['count'] results['total_story_count'] = total_stories return jsonify({'results': results})
def topic_update(topics_id): user_mc = user_admin_mediacloud_client() # top five cannot be empty fyi args = { 'name': request.form['name'], 'description': request.form['description'], 'solr_seed_query': request.form['solr_seed_query'], 'start_date': request.form['start_date'], 'end_date': request.form['end_date'], 'is_public': request.form['is_public'] if 'is_public' in request.form else None, 'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None, 'ch_monitor_id': request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0 and request.form['ch_monitor_id'] != 'null' else None, 'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None, 'max_stories': request.form['max_stories'] if 'max_stories' in request.form else None, 'twitter_topics_id': request.form['twitter_topics_id'] if 'twitter_topics_id' in request.form else None } # parse out any sources and collections to add media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]']) tag_ids_to_add = ids_from_comma_separated_str( request.form['collections[]']) # hack to support twitter-only topics if (len(media_ids_to_add) is 0) and (len(tag_ids_to_add) is 0): media_ids_to_add = None tag_ids_to_add = None result = user_mc.topicUpdate(topics_id, media_ids=media_ids_to_add, media_tags_ids=tag_ids_to_add, **args) return topic_summary( result['topics'][0]['topics_id'] ) # give them back new data, so they can update the client
def api_topics_preview_word_count(): user_mc = user_admin_mediacloud_client() solr_query = concatenate_query_for_solr( solr_seed_query=request.form['q'], media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None, tags_ids=ids_from_comma_separated_str(request.form['collections[]']) ) if 'collections[]' in request.form else None, fq = concatenate_solr_dates(start_date=request.form['start_date'], end_date=request.form['end_date']) word_count_result = user_mc.wordCount(solr_query=solr_query, solr_filter=fq) return jsonify(word_count_result ) # give them back new data, so they can update the client
def api_topics_preview_story_sample(): user_mc = user_mediacloud_client() solr_query = concatenate_query_for_solr( solr_seed_query=request.form['q'], media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None, tags_ids=ids_from_comma_separated_str(request.form['collections[]']) ) if 'collections[]' in request.form else None, fq = concatenate_solr_dates(start_date=request.form['start_date'], end_date=request.form['end_date']) story_count_result = user_mc.storyList(solr_query=solr_query, solr_filter=fq, sort=user_mc.SORT_RANDOM) return jsonify(story_count_result)
def api_topics_preview_split_story_count(): user_mc = user_admin_mediacloud_client() solr_query = concatenate_query_for_solr( solr_seed_query=request.form['q'], media_ids=ids_from_comma_separated_str(request.form['sources[]']) if 'sources[]' in request.form else None, tags_ids=ids_from_comma_separated_str(request.form['collections[]']) ) if 'collections[]' in request.form else None, fq = concatenate_solr_dates(start_date=request.form['start_date'], end_date=request.form['end_date']) results = user_mc.storyCount(solr_query=solr_query, solr_filter=fq, split=True) total_stories = 0 for c in results['counts']: total_stories += c['count'] results['total_story_count'] = total_stories return jsonify({'results': results})
def topic_create(): user_mc = user_admin_mediacloud_client() name = request.form['name'] description = request.form['description'] solr_seed_query = request.form['solr_seed_query'] start_date = request.form['start_date'] end_date = request.form['end_date'] optional_args = { 'is_public': request.form['is_public'] if 'is_public' in request.form else None, 'is_logogram': request.form['is_logogram'] if 'is_logogram' in request.form else None, 'ch_monitor_id': request.form['ch_monitor_id'] if len(request.form['ch_monitor_id']) > 0 and request.form['ch_monitor_id'] != 'null' else None, 'max_iterations': request.form['max_iterations'] if 'max_iterations' in request.form else None, 'max_stories': request.form['max_stories'] if 'max_stories' in request.form and request.form['max_stories'] != 'null' else None, } # parse out any sources and collections to add media_ids_to_add = ids_from_comma_separated_str(request.form['sources[]']) tag_ids_to_add = ids_from_comma_separated_str(request.form['collections[]']) try: topic_result = user_mc.topicCreate(name=name, description=description, solr_seed_query=solr_seed_query, start_date=start_date, end_date=end_date, media_ids=media_ids_to_add, media_tags_ids=tag_ids_to_add, **optional_args)['topics'][0] topic_id = topic_result['topics_id'] logger.info("Created new topic \"{}\" as {}".format(name, topic_id)) if set(tag_ids_to_add).intersection(US_COLLECTIONS): create_retweet_partisanship_focal_set(topic_result['topics_id']) spider_job = user_mc.topicSpider(topic_id) # kick off a spider, which will also generate a snapshot logger.info(" spider result = {}".format(json.dumps(spider_job))) results = user_mc.topic(topic_id) results['spider_job_state'] = spider_job return jsonify(results) # give them back new data, so they can update the client except Exception as e: logging.error("Topic creation failed {}".format(name)) logging.exception(e) return json_error_response(e.message, e.status_code)