def count_stories(q, fq): return mc.storyCount(q, fq)['count']
log = logging.getLogger(__name__) log.info( "---------------------------------------------------------------------------" ) start_time = time.time() requests_logger = logging.getLogger('requests') requests_logger.setLevel(logging.INFO) mc_logger = logging.getLogger('mediacloud') mc_logger.setLevel(logging.INFO) log.info("Using redis db %s as a cache" % config.get('cache', 'redis_db_number')) log.info("Working from controversy %s" % CONTROVERSY_ID) results = mc.storyCount("{~ topic:" + CONTROVERSY_ID + "}") log.info(" %s total stories" % results) data = incidentsv4.get_all() custom_query_keywords = incidentsv4.get_query_adjustments() @cache def fetch_all_stories(solr_query, solr_filter=''): log.info('Fetching stories for query {0}'.format(solr_query)) start = 0 offset = 500 all_stories = [] page = 0 while True: log.debug(" querying for %s | %s" % (solr_query, solr_filter))
def count_stories(q,fq): return mc.storyCount(q,fq)['count']
filename=os.path.join(basedir, "logs", config.get("spreadsheet", "year") + "list-all-stories.log"), level=logging.DEBUG, ) log = logging.getLogger(__name__) log.info("---------------------------------------------------------------------------") start_time = time.time() requests_logger = logging.getLogger("requests") requests_logger.setLevel(logging.INFO) mc_logger = logging.getLogger("mediacloud") mc_logger.setLevel(logging.INFO) log.info("Using redis db %s as a cache" % config.get("cache", "redis_db_number")) log.info("Working from controversy %s" % CONTROVERSY_ID) results = mc.storyCount("{~ topic:" + CONTROVERSY_ID + "}") log.info(" %s total stories" % results) data = incidentsv4.get_all() custom_query_keywords = incidentsv4.get_query_adjustments() @cache def fetch_all_stories(solr_query, solr_filter=""): log.info("Fetching stories for query {0}".format(solr_query)) start = 0 offset = 500 all_stories = [] page = 0 while True: log.debug(" querying for %s | %s" % (solr_query, solr_filter))
logging.basicConfig(filename=os.path.join(basedir,'logs', YEAR+'count-coverage.log'),level=logging.DEBUG) log = logging.getLogger(__name__) log.info("---------------------------------------------------------------------------") start_time = time.time() requests_logger = logging.getLogger('requests') requests_logger.setLevel(logging.INFO) mc_logger = logging.getLogger('mediacloud') mc_logger.setLevel(logging.INFO) log.info("Using redis db %s as a cache" % config.get('cache','redis_db_number')) log.info("Working from controversy %s" % CONTROVERSY_ID) controversy_filter = "{~ topic:"+CONTROVERSY_ID+"}" results = mc.storyCount(controversy_filter) log.info(" %s total stories" % CONTROVERSY_ID) # load the queries we wrote already our_query = None control_query = None with open(os.path.join(dest_dir,"query-with-names.txt"), "r") as text_file: our_query = controversy_filter +" AND ("+text_file.read()+")" with open(os.path.join(dest_dir,"query-no-names.txt"), "r") as text_file: control_query = text_file.read() log.info("Loaded both queries") log.info("Counting:") log.info(" Counting our sentences...") our_counts = mc.sentenceCount(our_query, split=True, split_start_date=YEAR+"-01-01", split_end_date=str(int(YEAR)+1)+"-01-01")