def main():
utils.setup_logger(__file__)
logging.info('update_relations_with_object_property.py -- starting')
search_collection = SolrCollection(os.getenv('SOLR_COLLECTION_SEARCH'))
property_to_relation = utils.load_resource('property_to_relation')
updates = []
for object_type, mapping in property_to_relation.items():
object_uri_to_source_mapping = get_object_uri_to_source_mapping(
search_collection, object_type, mapping['source'])
for relation in mapping['relations']:
relation_objects = search_collection.select_all_documents(
'sys_type:{0} AND {1}:[* TO *]'.format(relation['type'],
relation['match']),
['sys_uri', relation['match']],
id_field='sys_id')
updates += get_relation_updates(relation, mapping,
object_uri_to_source_mapping,
relation_objects)
search_collection.index_documents(updates)
logging.info('update_relations_with_object_property.py -- finished')
def manage_stopwords(collection: SolrCollection, language_code: str) -> None:
logging.info('managing stopwords_{0} resource'.format(language_code))
language_map = {
'nl': 'dutch',
'en': 'english',
}
language = language_map[language_code] if language_code in language_map \
else language_code
current_stopwords = collection.select_managed_stopwords(language)
logging.info('current stopwords: %s', len(current_stopwords))
stopwords = list(
set(utils.load_resource('stopwords_{0}'.format(
language_code))).difference(current_stopwords))
logging.info('words to add: %s', len(stopwords))
if len(stopwords) > 0:
if collection.add_managed_stopwords(language, stopwords):
logging.info('stopwords_{0} updated'.format(language_code))
else:
logging.error('failed to update stopwords_{0} resource'.format(
language_code))
else:
logging.info('no action required')
def update_relations(searcher: SolrCollection) -> None:
has_relations = utils.load_resource('has_relations')
updates = {}
for relation_source, mapping in has_relations.items():
logging.info('relations for %s', relation_source)
sources = searcher.select_all_documents(
fq='sys_type:{0}'.format(relation_source), id_field='sys_id')
rels = searcher.select_all_documents(
fl=list(set(list(mapping.values()) + ['sys_uri', 'sys_type'])),
fq='sys_type:{0}'.format(' OR sys_type:'.join(mapping.keys())),
id_field='sys_id')
logging.info(' subjects: %s', len(sources))
logging.info(' relations: %s', len(rels))
[source.update({'related_to': []}) for source in sources]
for source in sources:
related_to = set()
for mapping_target, mapping_source in mapping.items():
if mapping_target in source['related_to']:
continue
for relation in rels:
if mapping_target != relation['sys_type']:
continue
try:
if isinstance(relation[mapping_source], list):
if source['sys_uri'] in relation[mapping_source]:
related_to.add(mapping_target)
else:
if source['sys_uri'] == relation[mapping_source]:
related_to.add(mapping_target)
except KeyError:
continue
if source['sys_id'] not in updates:
updates[source['sys_id']] = set()
for related_to_type in related_to:
updates[source['sys_id']].add(related_to_type)
logging.info('indexing relations')
updates = [{
'sys_id': sys_id,
'related_to': {
'set': list(update)
}
} for sys_id, update in updates.items()]
searcher.index_documents(updates, commit=False)
logging.info('results')
logging.info(' indexed: %s', len(updates))
def main() -> None:
utils.setup_logger(__file__)
logging.info('list_downloader.py started')
lists = utils.load_resource('lists')
update_vocabularies(lists['vocabularies'])
update_taxonomies(lists['taxonomies'])
logging.info('list_downloader.py finished')
def manage_label_synonyms(collection: SolrCollection,
language_code: str) -> None:
logging.info('> managing label resource')
current_label_synonyms = collection.select_managed_synonyms(
'label_{0}'.format(language_code))
logging.info(' current: %s synonyms', len(current_label_synonyms))
label_synonyms = utils.load_resource('labels_{0}'.format(language_code))
synonyms_to_add = {
key: value
for key, value in label_synonyms.items()
if key not in current_label_synonyms.keys()
}
logging.info(' adding: %s synonyms', len(synonyms_to_add))
if len(synonyms_to_add) > 0:
collection.add_managed_synonyms('label_{0}'.format(language_code),
synonyms_to_add)
def manage_hierarchy_theme(collection: SolrCollection) -> None:
logging.info('> managing hierarchy_theme resources')
for hierarchy_item in ['hierarchy_theme', 'hierarchy_theme_query']:
logging.info('managed resource: synonyms|{0}'.format(hierarchy_item))
hierarchy_data = utils.load_resource(hierarchy_item)
current_data = collection.select_managed_synonyms(hierarchy_item)
if not current_data:
current_data = {}
logging.info(' current: %s synonyms', len(current_data))
data_to_add = {
key: value
for key, value in hierarchy_data.items()
if key not in current_data.keys()
}
logging.info(' adding: %s synonyms', len(data_to_add))
if len(data_to_add) > 0:
collection.add_managed_synonyms(hierarchy_item, data_to_add)
def main() -> None:
utils.setup_logger(__file__)
logging.info('generate_suggestions.py -- starting')
suggest = SolrCollection(os.getenv('SOLR_COLLECTION_SUGGESTER'))
search = SolrCollection(os.getenv('SOLR_COLLECTION_SEARCH'))
logging.info('clearing suggestions')
suggest.delete_documents('*:*', commit=False)
relation_counts = search.get_facet_counts('relation')
community_uri_to_name = search.select_all_documents(
fq='sys_type:community',
fl=['sys_uri', 'sys_name'],
id_field='sys_id'
)
community_uri_to_name = {community['sys_uri']: community['sys_name']
for community in community_uri_to_name}
suggestion_types = utils.load_resource('suggestions')
doc_suggestions = {doc_type: get_doc_suggestions(
search, doc_type, config['mapping'], relation_counts,
community_uri_to_name)
for doc_type, config in suggestion_types.items()}
logging.info('adding title suggestions:')
for doc_type, doc_type_suggestions in doc_suggestions.items():
suggest.index_documents(doc_type_suggestions, commit=False)
logging.info(' titles: %s of type %s',
len(doc_type_suggestions), doc_type)
user_defined_synonym_suggestions = {doc_type: get_doc_suggestions(
search, doc_type, config['user_defined_synonyms'], relation_counts,
community_uri_to_name, 'user_defined_synonyms:[* TO *]')
for doc_type, config in suggestion_types.items()
if 'user_defined_synonyms' in config}
logging.info('adding user defined synonym suggestions:')
for doc_type, doc_type_suggestions in user_defined_synonym_suggestions.items():
suggest.index_documents(doc_type_suggestions, commit=False)
logging.info(' user defined synonyms: %s of type %s',
len(doc_type_suggestions), doc_type)
context_suggestions = {
doc_type: {
relation: get_suggestions(search, doc_type, relation,
suggestion_types[relation]['mapping'],
community_uri_to_name)
} for doc_type, config in suggestion_types.items()
for relation in config['relations']}
logging.info('adding context suggestions:')
for doc_type, relations in context_suggestions.items():
for relation, suggestions in relations.items():
suggest.index_documents(suggestions, commit=False)
logging.info(' titles: %s of type %s in context of %s',
len(suggestions), relation, doc_type)
logging.info('adding theme suggestions:')
theme_suggestions = get_theme_suggestions(search, 'dataset')
suggest.index_documents(theme_suggestions, commit=False)
logging.info(' themes: %s in context of %s',
len(theme_suggestions), 'dataset')
logging.info('committing changes to index')
suggest.index_documents([], commit=True)
logging.info('building Solr suggester')
suggest.build_suggestions('build_suggest')
logging.info('generate_suggestions.py -- finished')
def update_reverse_relations(searcher: SolrCollection) -> None:
"""
Ensures that the relations in the following example are mirrored:
[{
"sys_type": "A",
"identifier": "Foo",
"relation_B": "Bar"
},
{
"sys_type": "B",
"identifier": "Bar"
}]
Afterwards the relations are set as:
[{
"sys_type": "A",
"identifier": "Foo",
"relation_B": "Bar"
},
{
"sys_type": "B",
"identifier": "Bar",
"relation_A": "Foo"
}]
This ensures that all relations are traversable regardless which object is
used as a reference point.
:param SolrCollection searcher: The searcher to find and update objects with
"""
relations = utils.load_resource('relations')
for source_object, source_data in relations.items():
for relation, mapping in source_data.items():
logging.info('updating reverse relations from %s to %s',
source_object, relation)
field_entities = searcher.select_all_documents(
'sys_type:{0}'.format(source_object),
['sys_id', mapping['match'], mapping['to']],
id_field='sys_id')
field_entities = {
entity[mapping['match']]: entity
for entity in field_entities
}
relation_entities = searcher.select_all_documents(
'sys_type:{0}'.format(relation),
[mapping['match'], mapping['from']],
id_field='sys_id')
entities_to_relation_entities = {}
for relation_entity in relation_entities:
if mapping['from'] not in relation_entity:
continue
for uri in relation_entity[mapping['from']]:
if uri in field_entities.keys():
if uri not in entities_to_relation_entities:
entities_to_relation_entities[uri] = []
entities_to_relation_entities[uri].append(
relation_entity[mapping['match']])
logging.info(
' found %s objects of type %s with relations to'
' objects of type %s',
len(entities_to_relation_entities.keys()), relation,
source_object)
deletes = [{
'sys_id': field_entity['sys_id'],
mapping['to']: {
'remove': field_entity[mapping['to']]
}
} for field_entity in field_entities
if mapping['to'] in field_entity
and field_entity[mapping['match']] not in
entities_to_relation_entities.keys()]
updates = [{
'sys_id': field_entities[uri]['sys_id'],
mapping['to']: {
'set': entities_to_relation_entities[uri]
}
} for uri in entities_to_relation_entities]
searcher.index_documents(deletes, commit=False)
searcher.index_documents(updates, commit=False)
logging.info('results')
logging.info(' deleted: %s', len(deletes))
logging.info(' updated: %s', len(updates))