Пример #1
0
 def run(self):
     """
     Drops all indices
     """
     global_coll = solr.global_collection()
     for doc in solr.get_all_docs_by_query(global_coll, '*:*', fields='id'):
         solr.collection_for_wiki(doc['id']).drop()
     global_coll.drop()
     solr.all_pages_collection().drop()
     solr.all_topics_collection().drop()
     solr.all_user_pages_collection().drop()
     solr.wiki_user_collection().drop()
     solr.user_collection().drop()
Пример #2
0
def aggregate_global_topic(topic):
    collection = solr.all_topics_collection()

    total_authorities = []
    all_user_id_dict = {}
    all_user_name_dict = {}
    all_wikis = []

    for doc in solr.get_all_docs_by_query(collection, topic):
        total_authorities.append(doc['total_authority_f'])
        if 'user_id_is' in doc:
            for user_id in doc['user_id_is']:
                all_user_id_dict[user_id] = True
        if 'user_names_ss' in doc:
            for user_name in doc['user_names_ss']:
                all_user_name_dict[user_name] = True
        if 'wiki_id_i' in doc:
            all_wikis.append(doc['wiki_id_i'])

    total_authority = sum(total_authorities)

    avg_authority = 0
    if total_authority > 0:
        avg_authority = total_authority / float(total_authority)

    return {
        'id': topic,
        'topic_s': {'set': topic},
        'wikis_is': {'set': all_wikis},
        'user_ids_is': {'set': all_user_id_dict.keys()},
        'user_names_ss': {'set': all_user_name_dict.keys()},
        'total_authority_f': {'set': total_authority},
        'avg_authority_f': {'set': avg_authority},
        'type_s': {'set': 'GlobalTopic'}
    }
Пример #3
0
 def search(**kwargs):
     return solr.get_docs_by_query_with_limit(solr.all_topics_collection(),
                                              kwargs['q'],
                                              boost='total_authority_f',
                                              fields=','.join(TopicModel.fields),
                                              qf='topic_txt_en',
                                              **sans_q(kwargs))
Пример #4
0
 def run(self):
     """
     Drops all indices
     """
     
     global_coll = solr.global_collection()
     print 'global'
     global_coll.optimize()
     for doc in solr.get_all_docs_by_query(global_coll, '*:*', fields='id'):
         print doc['id']
         solr.collection_for_wiki(doc['id']).optimize()
     print 'all pages'
     solr.all_pages_collection().optimize()
     print 'all topics'
     solr.all_topics_collection().optimize()
     print 'all user pages'
     solr.all_user_pages_collection().optimize()
     print 'wiki user'
     solr.wiki_user_collection().optimize()
     print 'user'
     solr.user_collection().optimize()
Пример #5
0
def analyze_topics_globally():
    print "Analyzing Topics..."
    collection = solr.all_topics_collection()

    se = SearchOptions()
    se.commonparams.q('*:*')

    futures = group(aggregate_global_topic.s(topic)
                    for topic, _ in solr.iterate_per_facetfield_value(collection, se, 'topic_s'))()

    while not futures.ready():
        print "Progress: (%d/%d)" % (futures.completed_count(), len(futures.results))
        sleep(2)

    collection.add(get_with_backoff(futures, []))
    collection.commit()
Пример #6
0
def ingest_data(wiki_id):
    """
    Create Solr documents for a given wiki ID

    :param wiki_id: the ID of the wiki (int or str)
    :type wiki_id: int
    :return:
    """
    # make sure all pages and all user pages exists
    solr.existing_collection(solr.all_pages_collection())
    solr.existing_collection(solr.all_user_pages_collection())

    resp = requests.get(u'http://www.wikia.com/api/v1/Wikis/Details', params={u'ids': wiki_id})
    items = resp.json()['items']
    if wiki_id not in items:
        print u"Wiki doesn't exist?"
        return

    api_data = items[wiki_id]
    wiki_data = {
        'id': api_data['id'],
        'wam_f': {'set': api_data['wam_score']},
        'title_s': {'set': api_data['title']},
        'attr_title': {'set': api_data['title']},
        'attr_desc': {'set': api_data['desc']}
    }
    for key in api_data['stats'].keys():
        wiki_data['%s_i' % key] = {'set': api_data['stats'][key]}

    wiki_api_data = requests.get(u'%swikia.php' % (api_data[u'url']),
                                 params={u'method': u'getForWiki',
                                         u'service': u'CrossWikiCore',
                                         u'controller': u'WikiaSearchIndexerController'}).json()[u'contents']

    wiki_data[u'hub_s'] = wiki_api_data[u'hub_s']
    
    # easier
    api_data[u'hub_s'] = wiki_api_data[u'hub_s']

    collection = solr.existing_collection(solr.collection_for_wiki(wiki_id))

    use_caching(is_read_only=True, shouldnt_compute=True)

    wpe = WikiPageToEntitiesService().get_value(wiki_id)
    if not wpe:
        print u"NO WIKI PAGE TO ENTITIES SERVICE FOR", wiki_id
        return False

    documents = []

    grouped_futures = []

    pages_to_authority = WikiAuthorityService().get_value(str(wiki_data['id']))
    for counter, (doc_id, entity_data) in enumerate(wpe.items()):
        documents.append({
            'id': doc_id,
            'attr_entities': {'set': list(set(entity_data.get(u'redirects', {}).values()
                                              + entity_data.get(u'titles')))},
            'type_s': {'set': 'Page'},
            'authority_f': {'set': pages_to_authority.get(doc_id, 0)},
            'hub_s': wiki_api_data['hub_s']
        })

        if counter != 0 and counter % 1500 == 0:
            grouped_futures.append(
                group(add_with_metadata.s(api_data, grouping) for grouping in iter_grouper(15, documents))()
            )

            documents = []

    grouped_futures.append(
        group(add_with_metadata.s(api_data, grouping) for grouping in iter_grouper(15, documents))()
    )

    # block on completion of all grouped futures
    completed = 0
    total = 0
    while len(filter(lambda x: not x.ready(), grouped_futures)) > 1:
        new_completed = 0
        new_total = 0
        for future in grouped_futures:
            new_completed += future.completed_count()
            new_total += len(future.results)
        if completed != new_completed or total != new_total:
            completed = new_completed
            total = new_total
            print "Grouped Tasks: (%d/%d)" % (completed, total)
        sleep(2)

    all_user_tuples = []
    for future in grouped_futures:
        result = get_with_backoff(future, [])
        map(all_user_tuples.extend, result)

    all_user_tuples = list(set(all_user_tuples))
    if not all_user_tuples:
        print "Empty user tuples, bailing"
        return

    # assign the unique user ids to the first variable, and the unique usernames to the second
    all_user_ids, all_users = zip(*all_user_tuples)

    collection.commit()
    solr.all_pages_collection().commit()
    solr.all_user_pages_collection().commit()

    wiki_data['attr_entities'] = {'set': []}

    for count, entities in WikiEntitiesService().get_value(str(wiki_id)).items():
        for entity in entities:
            map(wiki_data['attr_entities']['set'].append, [entity] * int(count))  # goddamnit count isn't int

    wiki_data['user_ids_is'] = {'set': all_user_ids}
    wiki_data['attr_users'] = {'set': all_users}
    wiki_data['total_authority_f'] = {'set': sum(pages_to_authority.values())}
    wiki_data['authorities_fs'] = {'set': pages_to_authority.values()}

    wiki_collection = solr.existing_collection(solr.global_collection())
    wiki_collection.add([wiki_data])
    wiki_collection.commit()
    print "Committed wiki data"

    print "Retrieving user docs..."
    futures = group(build_wiki_user_doc.s(api_data, user_tuple) for user_tuple in all_user_tuples)()
    future_result_len = len(futures.results)
    while not futures.ready():
        print "Progress: (%d/%d)" % (futures.completed_count(), future_result_len)
        sleep(2)

    user_docs = get_with_backoff(futures, [])
    if not user_docs:
        print "User docs was empty. Possibly connection problems."
        return

    authority_scaler = MinMaxScaler([doc['total_page_authority_f']['set'] for doc in user_docs])
    contribs_scaler = MinMaxScaler([doc['total_contribs_f']['set'] for doc in user_docs])
    for doc in user_docs:
        scaled_authority = authority_scaler.scale(doc['total_page_authority_f']['set'])
        scaled_contribs = contribs_scaler.scale(doc['total_contribs_f']['set'])
        doc['scaled_authority_f'] = {'set': scaled_authority}
        doc['scaled_contribs_f'] = {'set': scaled_contribs}
        doc['scaled_contribs_authority_f'] = {'set': scaled_authority * scaled_contribs}

    wiki_user_collection = solr.existing_collection(solr.wiki_user_collection())
    wiki_user_collection.add(user_docs)
    wiki_user_collection.commit()

    print "Analyzing topics"
    futures = group(get_wiki_topic_doc.s(wiki_data['id'], topic)
                    for topic in list(set(wiki_data['attr_entities']['set'])))()
    future_result_len = len(futures.results)
    counter = 0
    while not futures.ready():
        if counter % 5 == 0:
            print "Progress: (%d/%d)" % (futures.completed_count(), future_result_len)
        sleep(2)
        counter += 1
    topic_docs = get_with_backoff(futures, [])
    if not topic_docs:
        print "No topics, probably a connection error"
        return

    collection.add(topic_docs)
    collection.commit()

    topic_collection = solr.existing_collection(solr.all_topics_collection())
    topic_collection.add(topic_docs)
    topic_collection.commit()