def api_collection_sources_feed_status_csv(collection_id, source_type): user_mc = user_mediacloud_client() collection = user_mc.tag(collection_id) list_type = str(source_type).lower() media_in_collection = media_with_tag(user_mediacloud_key(), collection_id) media_info_in_collection = _media_list_edit_job.map(media_in_collection) if list_type == 'review': filtered_media = [ m for m in media_info_in_collection if m['active_feed_count'] > 0 and m['num_stories_90'] == 0 and m['num_stories_last_year'] > 0 ] elif list_type == 'remove': filtered_media = [ m for m in media_info_in_collection if m['active_feed_count'] > 0 and m['num_stories_90'] == 0 and m['num_stories_last_year'] == 0 and m['latest_scrape_job.state'] == 'failed' ] elif list_type == 'unscrapeable': filtered_media = [ m for m in media_info_in_collection if m['active_feed_count'] == 0 and m['num_stories_90'] > 0 ] elif list_type == 'working': filtered_media = [ m for m in media_info_in_collection if m['active_feed_count'] > 0 and m['num_stories_last_year'] > 0 ] else: filtered_media = media_info_in_collection file_prefix = "Collection {} ({}) - sources feed {}".format( collection_id, collection['tag'], source_type) properties_to_include = SOURCE_FEED_LIST_CSV_PROPS return csv.download_media_csv(filtered_media, file_prefix, properties_to_include)
def api_collection_sources_csv(collection_id): user_mc = user_mediacloud_client() collection = user_mc.tag(collection_id) # not cached because props can change often all_media = media_with_tag(user_mediacloud_key(), collection_id) file_prefix = "Collection {} ({}) - sources ".format(collection_id, collection['tag']) properties_to_include = SOURCE_LIST_CSV_EDIT_PROPS return csv.download_media_csv(all_media, file_prefix, properties_to_include)
def _stream_media_list_csv(user_mc_key, filename, topics_id, **kwargs): # Helper method to stream a list of media back to the client as a csv. Any args you pass in will be # simply be passed on to a call to topicMediaList. all_media = [] more_media = True params = kwargs params[ 'limit'] = 1000 # an arbitrary value to let us page through with big pages try: while more_media: page = apicache.topic_media_list(user_mediacloud_key(), topics_id, **params) media_list = page['media'] all_media = all_media + media_list if 'next' in page['link_ids']: params['link_id'] = page['link_ids']['next'] more_media = True else: more_media = False return csv.download_media_csv(all_media, filename, TOPIC_MEDIA_CSV_PROPS) except Exception as exception: return json.dumps({'error': str(exception)}, separators=(',', ':')), 400
def api_collection_sources_csv(collection_id): user_mc = user_mediacloud_client() collection = user_mc.tag(collection_id) # not cached because props can change often all_media = media_with_tag(user_mediacloud_key(), collection_id) file_prefix = "Collection {} ({}) - sources ".format(collection_id, collection['tag']) properties_to_include = SOURCE_LIST_CSV_EDIT_PROPS return csv.download_media_csv(all_media, file_prefix, properties_to_include)
def api_collection_sources_csv(collection_id): user_mc = user_admin_mediacloud_client() # info = user_mc.tag(int(collection_id)) all_media = media_with_tag(user_mediacloud_key(), collection_id) for src in all_media: for tag in src['media_source_tags']: if is_metadata_tag_set(tag['tag_sets_id']): format_metadata_fields(src, tag['tag_sets_id'], tag['tag']) file_prefix = "Collection_Sourcelist_Template_for_" + collection_id + "_" what_type_download = COLLECTIONS_TEMPLATE_PROPS_EDIT return csv.download_media_csv(all_media, file_prefix, what_type_download)
def api_collection_sources_csv(collection_id): user_mc = user_mediacloud_client() collection = user_mc.tag( collection_id) # not cached because props can change often all_media = media_with_tag(user_mediacloud_key(), collection_id) for src in all_media: for tag in src['media_source_tags']: if is_metadata_tag_set(tag['tag_sets_id']): format_metadata_fields(src, tag) file_prefix = "Collection {} ({}) - sources ".format( collection_id, collection['tag']) properties_to_include = COLLECTIONS_TEMPLATE_PROPS_EDIT return csv.download_media_csv(all_media, file_prefix, properties_to_include)
def api_collection_sources_feed_status_csv(collection_id, source_type): user_mc = user_mediacloud_client() collection = user_mc.tag(collection_id) type = str(source_type).lower() media_in_collection = media_with_tag(user_mediacloud_key(), collection_id) media_info_in_collection = _fetch_collection_source_feed_info(media_in_collection) if type == 'review': filtered_media = [m for m in media_info_in_collection if m['active_feed_count'] > 0 and m['num_stories_90'] == 0 and m['num_stories_last_year'] > 0] elif type == 'remove': filtered_media = [m for m in media_info_in_collection if m['active_feed_count'] > 0 and m['num_stories_90'] == 0 and m['num_stories_last_year'] == 0 and m['latest_scrape_job.state'] == 'failed'] elif type == 'unscrapeable': filtered_media = [m for m in media_info_in_collection if m['active_feed_count'] == 0 and m['num_stories_90'] > 0] elif type == 'working': filtered_media = [m for m in media_info_in_collection if m['active_feed_count'] > 0 and m['num_stories_last_year'] > 0] else: filtered_media = media_info_in_collection file_prefix = "Collection {} ({}) - sources feed {}".format(collection_id, collection['tag'], source_type) properties_to_include = SOURCE_FEED_LIST_CSV_PROPS return csv.download_media_csv(filtered_media, file_prefix, properties_to_include)
def _stream_media_list_csv(user_mc_key, filename, topics_id, **kwargs): # Helper method to stream a list of media back to the client as a csv. Any args you pass in will be # simply be passed on to a call to topicMediaList. add_metadata = False # off for now because this is SUPER slow all_media = [] more_media = True params = kwargs params[ 'limit'] = 1000 # an arbitrary value to let us page through with big pages try: cols_to_export = TOPICS_TEMPLATE_PROPS if not add_metadata: cols_to_export = cols_to_export[:-4] # remove the metadata cols while more_media: page = apicache.topic_media_list(user_mediacloud_key(), topics_id, **params) media_list = page['media'] user_mc = user_admin_mediacloud_client() if add_metadata: for media_item in media_list: media_info = user_mc.media(media_item['media_id']) for eachItem in media_info['media_source_tags']: if is_metadata_tag_set(eachItem['tag_sets_id']): format_metadata_fields(media_item, eachItem) all_media = all_media + media_list if 'next' in page['link_ids']: params['link_id'] = page['link_ids']['next'] more_media = True else: more_media = False return csv.download_media_csv(all_media, filename, cols_to_export) except Exception as exception: return json.dumps({'error': str(exception)}, separators=(',', ':')), 400
def _stream_media_list_csv(user_mc_key, filename, topics_id, **kwargs): # Helper method to stream a list of media back to the client as a csv. Any args you pass in will be # simply be passed on to a call to topicMediaList. all_media = [] more_media = True params = kwargs params['limit'] = 1000 # an arbitrary value to let us page through with big pages try: while more_media: page = apicache.topic_media_list(user_mediacloud_key(), topics_id, **params) media_list = page['media'] all_media = all_media + media_list if 'next' in page['link_ids']: params['link_id'] = page['link_ids']['next'] more_media = True else: more_media = False return csv.download_media_csv(all_media, filename, TOPIC_MEDIA_CSV_PROPS) except Exception as exception: return json.dumps({'error': str(exception)}, separators=(',', ':')), 400
def download_sources_csv(all_media, file_prefix): if user_has_auth_role(ROLE_MEDIA_EDIT): what_type_download = SOURCES_TEMPLATE_PROPS_EDIT else: what_type_download = SOURCES_TEMPLATE_PROPS_VIEW # no editor_notes return download_media_csv(all_media, file_prefix, what_type_download)