def story_docs_csv(keywords, media, start, end):
user_mc = mcapi.MediaCloud(flask_login.current_user.get_id())
query = app.core.util.solr_query(app.core.util.media_to_solr(media), start, end)
all_stories = []
last_processed_stories_id = 0
more_stories = True
def csv_escape(s):
return '"%s"' % s.replace('"', '"",')
try:
while more_stories:
res = user_mc.storyList(keywords, query, last_processed_stories_id, 1000)
if len(res) > 0:
stories = [ [str(s['stories_id']),s['language'],csv_escape(s['title']),s['url'],s['publish_date']]
for s in res]
last_processed_stories_id = res[len(res)-1]['processed_stories_id']
all_stories = all_stories + stories
more_stories = True
else:
more_stories = False
def stream_csv(story_list):
yield ','.join(['stories_id','language','title','url','publish_date']) + '\n'
for story in story_list:
yield ','.join(story) + '\n'
download_filename = 'mediacloud-results-'+datetime.datetime.now().strftime('%Y%m%d%H%M%S')+'.csv'
return flask.Response(stream_csv(all_stories), mimetype='text/csv',
headers={"Content-Disposition":"attachment;filename="+download_filename})
except Exception as exception:
return json.dumps({'error':str(exception)}, separators=(',',':')), 400
def num_found_worker(arg):
mc_key, keywords, date, filter_query = arg
mc = mcapi.MediaCloud(mc_key)
query = "%s AND (%s)" % (keywords_to_solr(keywords), filter_query)
app.core.logger.debug("query: sentenceList: %s" % query)
res = mc.sentenceList(query, '', 0, 0)
return {'date': date, 'numFound': res['response']['numFound']}
requests_logger = logging.getLogger('requests')
requests_logger.propagate = False
requests_logger.addHandler(log_handler)
# set up mediameter logging the same way
logger.propagate = False
logger.addHandler(log_handler)
logger.info("---------------------------------------------------------------------------------------")
# Flask app
flapp = flask.Flask(__name__)
flapp.secret_key = 'put secret key here'
# Create media cloud api
mc_key = config.get('mediacloud','key')
mc = mcapi.MediaCloud(mc_key)
logger.info("Connected to MediaCloud with default key %s" % (mc_key))
#logging.getLogger('MediaCloud').setLevel(logging.DEBUG)
# Create user login manager
login_manager = flask_login.LoginManager()
login_manager.init_app(flapp)
# Connect to db
host = config.get('database', 'host')
database = config.get('database', 'database')
db = pymongo.Connection(host)[database]
logger.info("Connected to DB %s@%s" % (database,host))
# Set up routes and content
import app.core.views
def _get_user_mc(api_key, force_admin=False):
if force_admin or (api_key == app_mc_key):
return mcapi.AdminMediaCloud(api_key)
return mcapi.MediaCloud(api_key)