Example #1
0
def index():
    content = flask.render_template('core/progress.html')
    template = config.get('flask', 'template')
    return flask.render_template(
        template
        , content=content
        , google_analytics_id=config.get('analytics', 'google_analytics_id')
    )
Example #2
0
def index():
    content = flask.render_template('core/progress.html')
    template = config.get('flask', 'template')
    return flask.render_template(
        template
        , content=content
        , google_analytics_id=config.get('analytics', 'google_analytics_id')
    )
Example #3
0
 def results(self):
     if int(config.get('threading', 'num_threads')) > 0:
         return NumFound.thread_pool.map(num_found_worker, self.to_query)
     return [num_found_worker(arg) for arg in self.to_query]
Example #4
0
class NumFound:
    def __init__(self, mc_key, keywords, media, start, end):
        self.mc_key = mc_key
        self.to_query = []
        queries = solr_date_queries(media_to_solr(media), start, end)
        for q in queries:
            date, query = q
            #self.to_query.append((self, keywords, date, query))
            self.to_query.append((mc_key, keywords, date, query))
            
    def results(self):
        if int(config.get('threading', 'num_threads')) > 0:
            return NumFound.thread_pool.map(num_found_worker, self.to_query)
        return [num_found_worker(arg) for arg in self.to_query]

# This should be an instancemethod of NumFound, but Pool.map() requires it
# to be pickle-able, so this is a quick hack to work around that.
def num_found_worker(arg):
    mc_key, keywords, date, filter_query = arg
    mc = mcapi.MediaCloud(mc_key)
    query = "%s AND (%s)" % (keywords_to_solr(keywords), filter_query)
    logger.debug("query: sentenceList: %s" % query)
    res = mc.sentenceList(query, '', 0, 0)
    return {
        'date': date
        , 'numFound': res['response']['numFound']
    }

if int(config.get('threading', 'num_threads')) > 0:
    NumFound.thread_pool = multiprocessing.Pool(processes=int(config.get('threading', 'num_threads')))
Example #5
0
def index():
    content = flask.render_template('core/progress.html')
    template = config.get('flask', 'template')
    return flask.render_template(template, content=content)
Example #6
0
 def results(self):
     if int(config.get('threading', 'num_threads')) > 0:
         return NumFound.thread_pool.map(num_found_worker, self.to_query)
     return [num_found_worker(arg) for arg in self.to_query]
Example #7
0
class NumFound:
    def __init__(self, mc_key, keywords, media, start, end):
        self.mc_key = mc_key
        self.to_query = []
        queries = solr_date_queries(media_to_solr(media), start, end)
        for q in queries:
            date, query = q
            #self.to_query.append((self, keywords, date, query))
            self.to_query.append((mc_key, keywords, date, query))

    def results(self):
        if int(config.get('threading', 'num_threads')) > 0:
            return NumFound.thread_pool.map(num_found_worker, self.to_query)
        return [num_found_worker(arg) for arg in self.to_query]


# This should be an instancemethod of NumFound, but Pool.map() requires it
# to be pickle-able, so this is a quick hack to work around that.
def num_found_worker(arg):
    mc_key, keywords, date, filter_query = arg
    mc = mcapi.MediaCloud(mc_key)
    query = "%s AND (%s)" % (keywords_to_solr(keywords), filter_query)
    app.core.logger.debug("query: sentenceList: %s" % query)
    res = mc.sentenceList(query, '', 0, 0)
    return {'date': date, 'numFound': res['response']['numFound']}


if int(config.get('threading', 'num_threads')) > 0:
    NumFound.thread_pool = multiprocessing.Pool(
        processes=int(config.get('threading', 'num_threads')))