def main():
utils.setup_logger(__file__)
logging.info('update_relations_with_object_property.py -- starting')
search_collection = SolrCollection(os.getenv('SOLR_COLLECTION_SEARCH'))
property_to_relation = utils.load_resource('property_to_relation')
updates = []
for object_type, mapping in property_to_relation.items():
object_uri_to_source_mapping = get_object_uri_to_source_mapping(
search_collection, object_type, mapping['source'])
for relation in mapping['relations']:
relation_objects = search_collection.select_all_documents(
'sys_type:{0} AND {1}:[* TO *]'.format(relation['type'],
relation['match']),
['sys_uri', relation['match']],
id_field='sys_id')
updates += get_relation_updates(relation, mapping,
object_uri_to_source_mapping,
relation_objects)
search_collection.index_documents(updates)
logging.info('update_relations_with_object_property.py -- finished')
def update_authority_kind(searcher: SolrCollection) -> None:
organization_types = {
organization['sys_uri']: organization['kind']
for organization in searcher.select_all_documents(
'sys_type:organization', ['sys_uri', 'kind'], id_field='sys_id')
if 'kind' in organization and 'sys_uri' in organization
}
objects_with_authority = searcher.select_all_documents(
'authority:[* TO *]', ['sys_id, authority'], id_field='sys_id')
logging.info('Found {0} objects with a relation with an authority'.format(
len(objects_with_authority)))
updates = [{
'sys_id': donl_object['sys_id'],
'authority_kind': {
'set': [
organization_types[authority]
for authority in donl_object['authority']
if authority in organization_types
]
}
} for donl_object in objects_with_authority]
searcher.index_documents(updates, commit=False)
logging.info('results')
logging.info(' indexed: %s', len(updates))
def update_relations(searcher: SolrCollection) -> None:
has_relations = utils.load_resource('has_relations')
updates = {}
for relation_source, mapping in has_relations.items():
logging.info('relations for %s', relation_source)
sources = searcher.select_all_documents(
fq='sys_type:{0}'.format(relation_source), id_field='sys_id')
rels = searcher.select_all_documents(
fl=list(set(list(mapping.values()) + ['sys_uri', 'sys_type'])),
fq='sys_type:{0}'.format(' OR sys_type:'.join(mapping.keys())),
id_field='sys_id')
logging.info(' subjects: %s', len(sources))
logging.info(' relations: %s', len(rels))
[source.update({'related_to': []}) for source in sources]
for source in sources:
related_to = set()
for mapping_target, mapping_source in mapping.items():
if mapping_target in source['related_to']:
continue
for relation in rels:
if mapping_target != relation['sys_type']:
continue
try:
if isinstance(relation[mapping_source], list):
if source['sys_uri'] in relation[mapping_source]:
related_to.add(mapping_target)
else:
if source['sys_uri'] == relation[mapping_source]:
related_to.add(mapping_target)
except KeyError:
continue
if source['sys_id'] not in updates:
updates[source['sys_id']] = set()
for related_to_type in related_to:
updates[source['sys_id']].add(related_to_type)
logging.info('indexing relations')
updates = [{
'sys_id': sys_id,
'related_to': {
'set': list(update)
}
} for sys_id, update in updates.items()]
searcher.index_documents(updates, commit=False)
logging.info('results')
logging.info(' indexed: %s', len(updates))
def main():
utils.setup_logger(__file__)
logging.info('generate_relations.py -- starting')
collection = SolrCollection(os.getenv('SOLR_COLLECTION_SEARCH'))
update_reverse_relations(collection)
logging.info('committing index changes')
collection.index_documents([], commit=True)
update_relations(collection)
logging.info('committing index changes')
collection.index_documents([], commit=True)
update_authority_kind(collection)
logging.info('committing index changes')
collection.index_documents([], commit=True)
update_popularity(collection)
logging.info('committing index changes')
collection.index_documents([], commit=True)
logging.info('generate_relations.py -- finished')
def main() -> None:
utils.setup_logger(__file__)
logging.info('aggregate_signals.py started')
signal_collection = SolrCollection(os.getenv('SOLR_COLLECTION_SIGNALS'))
signal_aggregated_collection = SolrCollection(
os.getenv('SOLR_COLLECTION_SIGNALS_AGGREGATED'))
signals = signal_collection.select_all_documents()
aggregated_signals = signal_aggregated_collection.select_all_documents()
signal_aggregated_collection.index_documents(
get_aggregations(aggregate_fields(signals, ['query', 'handler']),
aggregated_signals, 'query'))
signal_aggregated_collection.index_documents(
get_aggregations(
aggregate_fields(
reduce_date_field_to_hours(signals, 'search_timestamp'),
['search_timestamp', 'handler']), aggregated_signals,
'search_timestamp'))
signal_aggregated_collection.index_documents(
get_aggregations(
aggregate_fields(preprocess_filters(signals),
['filters', 'handler']), aggregated_signals,
'filters'))
signal_collection.delete_documents('*:*')
logging.info('aggregate_signals.py finished')
def update_popularity(searcher: SolrCollection) -> None:
logging.info('Updating popularity')
relation_counts = searcher.get_facet_counts('relation')
donl_objects = searcher.select_all_documents(fl=['sys_uri', 'popularity'],
id_field='sys_id')
updates = [
{
'sys_id': donl_object['sys_id'],
'popularity': {
'set': relation_counts[donl_object['sys_uri']]
}
} for donl_object in donl_objects
if donl_object['sys_uri'] in relation_counts
and not ('popularity' in donl_object and donl_object['popularity'] ==
relation_counts[donl_object['sys_uri']])
]
searcher.index_documents(updates, commit=False)
logging.info('results')
logging.info(' indexed: %s', len(updates))
def main() -> None:
utils.setup_logger(__file__)
logging.info('generate_suggestions.py -- starting')
suggest = SolrCollection(os.getenv('SOLR_COLLECTION_SUGGESTER'))
search = SolrCollection(os.getenv('SOLR_COLLECTION_SEARCH'))
logging.info('clearing suggestions')
suggest.delete_documents('*:*', commit=False)
relation_counts = search.get_facet_counts('relation')
community_uri_to_name = search.select_all_documents(
fq='sys_type:community',
fl=['sys_uri', 'sys_name'],
id_field='sys_id'
)
community_uri_to_name = {community['sys_uri']: community['sys_name']
for community in community_uri_to_name}
suggestion_types = utils.load_resource('suggestions')
doc_suggestions = {doc_type: get_doc_suggestions(
search, doc_type, config['mapping'], relation_counts,
community_uri_to_name)
for doc_type, config in suggestion_types.items()}
logging.info('adding title suggestions:')
for doc_type, doc_type_suggestions in doc_suggestions.items():
suggest.index_documents(doc_type_suggestions, commit=False)
logging.info(' titles: %s of type %s',
len(doc_type_suggestions), doc_type)
user_defined_synonym_suggestions = {doc_type: get_doc_suggestions(
search, doc_type, config['user_defined_synonyms'], relation_counts,
community_uri_to_name, 'user_defined_synonyms:[* TO *]')
for doc_type, config in suggestion_types.items()
if 'user_defined_synonyms' in config}
logging.info('adding user defined synonym suggestions:')
for doc_type, doc_type_suggestions in user_defined_synonym_suggestions.items():
suggest.index_documents(doc_type_suggestions, commit=False)
logging.info(' user defined synonyms: %s of type %s',
len(doc_type_suggestions), doc_type)
context_suggestions = {
doc_type: {
relation: get_suggestions(search, doc_type, relation,
suggestion_types[relation]['mapping'],
community_uri_to_name)
} for doc_type, config in suggestion_types.items()
for relation in config['relations']}
logging.info('adding context suggestions:')
for doc_type, relations in context_suggestions.items():
for relation, suggestions in relations.items():
suggest.index_documents(suggestions, commit=False)
logging.info(' titles: %s of type %s in context of %s',
len(suggestions), relation, doc_type)
logging.info('adding theme suggestions:')
theme_suggestions = get_theme_suggestions(search, 'dataset')
suggest.index_documents(theme_suggestions, commit=False)
logging.info(' themes: %s in context of %s',
len(theme_suggestions), 'dataset')
logging.info('committing changes to index')
suggest.index_documents([], commit=True)
logging.info('building Solr suggester')
suggest.build_suggestions('build_suggest')
logging.info('generate_suggestions.py -- finished')
def update_reverse_relations(searcher: SolrCollection) -> None:
"""
Ensures that the relations in the following example are mirrored:
[{
"sys_type": "A",
"identifier": "Foo",
"relation_B": "Bar"
},
{
"sys_type": "B",
"identifier": "Bar"
}]
Afterwards the relations are set as:
[{
"sys_type": "A",
"identifier": "Foo",
"relation_B": "Bar"
},
{
"sys_type": "B",
"identifier": "Bar",
"relation_A": "Foo"
}]
This ensures that all relations are traversable regardless which object is
used as a reference point.
:param SolrCollection searcher: The searcher to find and update objects with
"""
relations = utils.load_resource('relations')
for source_object, source_data in relations.items():
for relation, mapping in source_data.items():
logging.info('updating reverse relations from %s to %s',
source_object, relation)
field_entities = searcher.select_all_documents(
'sys_type:{0}'.format(source_object),
['sys_id', mapping['match'], mapping['to']],
id_field='sys_id')
field_entities = {
entity[mapping['match']]: entity
for entity in field_entities
}
relation_entities = searcher.select_all_documents(
'sys_type:{0}'.format(relation),
[mapping['match'], mapping['from']],
id_field='sys_id')
entities_to_relation_entities = {}
for relation_entity in relation_entities:
if mapping['from'] not in relation_entity:
continue
for uri in relation_entity[mapping['from']]:
if uri in field_entities.keys():
if uri not in entities_to_relation_entities:
entities_to_relation_entities[uri] = []
entities_to_relation_entities[uri].append(
relation_entity[mapping['match']])
logging.info(
' found %s objects of type %s with relations to'
' objects of type %s',
len(entities_to_relation_entities.keys()), relation,
source_object)
deletes = [{
'sys_id': field_entity['sys_id'],
mapping['to']: {
'remove': field_entity[mapping['to']]
}
} for field_entity in field_entities
if mapping['to'] in field_entity
and field_entity[mapping['match']] not in
entities_to_relation_entities.keys()]
updates = [{
'sys_id': field_entities[uri]['sys_id'],
mapping['to']: {
'set': entities_to_relation_entities[uri]
}
} for uri in entities_to_relation_entities]
searcher.index_documents(deletes, commit=False)
searcher.index_documents(updates, commit=False)
logging.info('results')
logging.info(' deleted: %s', len(deletes))
logging.info(' updated: %s', len(updates))