def update_popularity(searcher: SolrCollection) -> None:
logging.info('Updating popularity')
relation_counts = searcher.get_facet_counts('relation')
donl_objects = searcher.select_all_documents(fl=['sys_uri', 'popularity'],
id_field='sys_id')
updates = [
{
'sys_id': donl_object['sys_id'],
'popularity': {
'set': relation_counts[donl_object['sys_uri']]
}
} for donl_object in donl_objects
if donl_object['sys_uri'] in relation_counts
and not ('popularity' in donl_object and donl_object['popularity'] ==
relation_counts[donl_object['sys_uri']])
]
searcher.index_documents(updates, commit=False)
logging.info('results')
logging.info(' indexed: %s', len(updates))
def main() -> None:
utils.setup_logger(__file__)
logging.info('generate_suggestions.py -- starting')
suggest = SolrCollection(os.getenv('SOLR_COLLECTION_SUGGESTER'))
search = SolrCollection(os.getenv('SOLR_COLLECTION_SEARCH'))
logging.info('clearing suggestions')
suggest.delete_documents('*:*', commit=False)
relation_counts = search.get_facet_counts('relation')
community_uri_to_name = search.select_all_documents(
fq='sys_type:community',
fl=['sys_uri', 'sys_name'],
id_field='sys_id'
)
community_uri_to_name = {community['sys_uri']: community['sys_name']
for community in community_uri_to_name}
suggestion_types = utils.load_resource('suggestions')
doc_suggestions = {doc_type: get_doc_suggestions(
search, doc_type, config['mapping'], relation_counts,
community_uri_to_name)
for doc_type, config in suggestion_types.items()}
logging.info('adding title suggestions:')
for doc_type, doc_type_suggestions in doc_suggestions.items():
suggest.index_documents(doc_type_suggestions, commit=False)
logging.info(' titles: %s of type %s',
len(doc_type_suggestions), doc_type)
user_defined_synonym_suggestions = {doc_type: get_doc_suggestions(
search, doc_type, config['user_defined_synonyms'], relation_counts,
community_uri_to_name, 'user_defined_synonyms:[* TO *]')
for doc_type, config in suggestion_types.items()
if 'user_defined_synonyms' in config}
logging.info('adding user defined synonym suggestions:')
for doc_type, doc_type_suggestions in user_defined_synonym_suggestions.items():
suggest.index_documents(doc_type_suggestions, commit=False)
logging.info(' user defined synonyms: %s of type %s',
len(doc_type_suggestions), doc_type)
context_suggestions = {
doc_type: {
relation: get_suggestions(search, doc_type, relation,
suggestion_types[relation]['mapping'],
community_uri_to_name)
} for doc_type, config in suggestion_types.items()
for relation in config['relations']}
logging.info('adding context suggestions:')
for doc_type, relations in context_suggestions.items():
for relation, suggestions in relations.items():
suggest.index_documents(suggestions, commit=False)
logging.info(' titles: %s of type %s in context of %s',
len(suggestions), relation, doc_type)
logging.info('adding theme suggestions:')
theme_suggestions = get_theme_suggestions(search, 'dataset')
suggest.index_documents(theme_suggestions, commit=False)
logging.info(' themes: %s in context of %s',
len(theme_suggestions), 'dataset')
logging.info('committing changes to index')
suggest.index_documents([], commit=True)
logging.info('building Solr suggester')
suggest.build_suggestions('build_suggest')
logging.info('generate_suggestions.py -- finished')