def count_stories(q, fq):
return mc.storyCount(q, fq)['count']
log = logging.getLogger(__name__)
log.info(
"---------------------------------------------------------------------------"
)
start_time = time.time()
requests_logger = logging.getLogger('requests')
requests_logger.setLevel(logging.INFO)
mc_logger = logging.getLogger('mediacloud')
mc_logger.setLevel(logging.INFO)
log.info("Using redis db %s as a cache" %
config.get('cache', 'redis_db_number'))
log.info("Working from controversy %s" % CONTROVERSY_ID)
results = mc.storyCount("{~ topic:" + CONTROVERSY_ID + "}")
log.info(" %s total stories" % results)
data = incidentsv4.get_all()
custom_query_keywords = incidentsv4.get_query_adjustments()
@cache
def fetch_all_stories(solr_query, solr_filter=''):
log.info('Fetching stories for query {0}'.format(solr_query))
start = 0
offset = 500
all_stories = []
page = 0
while True:
log.debug(" querying for %s | %s" % (solr_query, solr_filter))
def count_stories(q,fq):
return mc.storyCount(q,fq)['count']
filename=os.path.join(basedir, "logs", config.get("spreadsheet", "year") + "list-all-stories.log"),
level=logging.DEBUG,
)
log = logging.getLogger(__name__)
log.info("---------------------------------------------------------------------------")
start_time = time.time()
requests_logger = logging.getLogger("requests")
requests_logger.setLevel(logging.INFO)
mc_logger = logging.getLogger("mediacloud")
mc_logger.setLevel(logging.INFO)
log.info("Using redis db %s as a cache" % config.get("cache", "redis_db_number"))
log.info("Working from controversy %s" % CONTROVERSY_ID)
results = mc.storyCount("{~ topic:" + CONTROVERSY_ID + "}")
log.info(" %s total stories" % results)
data = incidentsv4.get_all()
custom_query_keywords = incidentsv4.get_query_adjustments()
@cache
def fetch_all_stories(solr_query, solr_filter=""):
log.info("Fetching stories for query {0}".format(solr_query))
start = 0
offset = 500
all_stories = []
page = 0
while True:
log.debug(" querying for %s | %s" % (solr_query, solr_filter))
logging.basicConfig(filename=os.path.join(basedir,'logs',
YEAR+'count-coverage.log'),level=logging.DEBUG)
log = logging.getLogger(__name__)
log.info("---------------------------------------------------------------------------")
start_time = time.time()
requests_logger = logging.getLogger('requests')
requests_logger.setLevel(logging.INFO)
mc_logger = logging.getLogger('mediacloud')
mc_logger.setLevel(logging.INFO)
log.info("Using redis db %s as a cache" % config.get('cache','redis_db_number'))
log.info("Working from controversy %s" % CONTROVERSY_ID)
controversy_filter = "{~ topic:"+CONTROVERSY_ID+"}"
results = mc.storyCount(controversy_filter)
log.info(" %s total stories" % CONTROVERSY_ID)
# load the queries we wrote already
our_query = None
control_query = None
with open(os.path.join(dest_dir,"query-with-names.txt"), "r") as text_file:
our_query = controversy_filter +" AND ("+text_file.read()+")"
with open(os.path.join(dest_dir,"query-no-names.txt"), "r") as text_file:
control_query = text_file.read()
log.info("Loaded both queries")
log.info("Counting:")
log.info(" Counting our sentences...")
our_counts = mc.sentenceCount(our_query, split=True,
split_start_date=YEAR+"-01-01", split_end_date=str(int(YEAR)+1)+"-01-01")
