def try_map_response_to_subject( response: Dict, type_name: str, language_keys: Optional[List[str]] = lang_keys, ) -> Dict: """Maps the default attributes which every subject has: qid, image, label, description, classes, wikipediaLink (including language specific attributes) Args: response: The wikidata entity which should be mapped to an openArtBrowser entity type_name: Type name of the entity language_keys: All language keys which should be extracted. Defaults to languageconfig.csv Returns: A dict of an openArtBrowser entity """ try: qid = response[ID] except Exception as error: logger.error("Error on qid, skipping item. Error: {0}".format(error)) return None # How to get image url # https://stackoverflow.com/questions/34393884/how-to-get-image-url-property-from-wikidata-item-by-api try: image = get_image_url_by_name( response[CLAIMS][PROPERTY_NAME_TO_PROPERTY_ID[IMAGE]][0][MAINSNAK] [DATAVALUE][VALUE]) except: image = "" label = map_wd_attribute.try_get_label_or_description( response, LABEL[PLURAL], EN, type_name) description = map_wd_attribute.try_get_label_or_description( response, DESCRIPTION[PLURAL], EN, type_name) classes = map_wd_attribute.try_get_qid_reference_list( response, PROPERTY_NAME_TO_PROPERTY_ID[CLASS[SINGULAR]], type_name) subject_dict = { ID: qid, CLASS[PLURAL]: classes, LABEL[SINGULAR]: label, DESCRIPTION[SINGULAR]: description, IMAGE: image, } for langkey in language_keys: label_lang = map_wd_attribute.try_get_label_or_description( response, LABEL[PLURAL], langkey, type_name) description_lang = map_wd_attribute.try_get_label_or_description( response, DESCRIPTION[PLURAL], langkey, type_name) wikipedia_link_lang = map_wd_attribute.try_get_wikipedia_link( response, langkey, type_name) subject_dict.update({ f"{LABEL[SINGULAR]}_{langkey}": label_lang, f"{DESCRIPTION[SINGULAR]}_{langkey}": description_lang, f"{WIKIPEDIA_LINK}_{langkey}": wikipedia_link_lang, }) return subject_dict
def get_exhibition_entities( qids: Set[str], language_keys: Optional[List[str]] = lang_keys, type_name: str = EXHIBITION, ) -> Dict[str, Dict]: """Function to get the exhibition entities from wikidata Args: qids: Distinct qid set to get the entities from language_keys: Language keys to extract label and description from. Defaults to languageconfig.csv type_name: OAB type name. Defaults to EXHIBITION. Returns: A dict with the qids as key and the JSON object as value """ print(datetime.datetime.now(), f"Starting with exhibition entities") print(f"Total exhibition entities to extract: {len(qids)}") item_count = 0 extract_dicts = {} chunk_size = 50 # The chunksize 50 is allowed by the wikidata api, bigger numbers need special permissions id_chunks = chunks(list(qids), chunk_size) for chunk in id_chunks: query_result = wikidata_entity_request(chunk) for result in query_result[ENTITIES].values(): try: qid = result[ID] except Exception as error: logger.error( "Error on qid, skipping item. Error: {0}".format(error)) continue label = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], EN, type_name) description = map_wd_attribute.try_get_label_or_description( result, DESCRIPTION[PLURAL], EN, type_name) start_time = map_wd_attribute.try_get_year_from_property_timestamp( result, PROPERTY_NAME_TO_PROPERTY_ID[START_TIME], type_name) end_time = map_wd_attribute.try_get_year_from_property_timestamp( result, PROPERTY_NAME_TO_PROPERTY_ID[END_TIME], type_name) extract_dicts.update({ qid: { LABEL[SINGULAR]: label, DESCRIPTION[SINGULAR]: description, START_TIME: start_time, END_TIME: end_time, TYPE: EXHIBITION, } }) for langkey in language_keys: label_lang = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], langkey, type_name) description_lang = map_wd_attribute.try_get_label_or_description( result, DESCRIPTION[PLURAL], langkey, type_name) extract_dicts[qid][f"{LABEL[SINGULAR]}_{langkey}"] = label_lang extract_dicts[qid][ f"{DESCRIPTION[SINGULAR]}_{langkey}"] = description_lang item_count += len(chunk) print( f"Status of exhibition entities: {item_count}/{len(qids)}", end="\r", flush=True, ) print(datetime.datetime.now(), f"Finished with exhibition entities") return extract_dicts
def get_entity_labels( type_name: str, qids: List[str], language_keys: Optional[List[str]] = lang_keys, ) -> List[Dict]: """Function to get the entity labels from wikidata Args: type_name: oab type e. g. movement qids: List of qids to extract the labels from language_keys: All language keys which should be extracted. Defaults to languageconfig.csv Returns: List of dicts containing the qid and the labels for each language """ print(datetime.datetime.now(), f"Starting with {type_name} {LABEL[PLURAL]}") print(f"Total {type_name} {LABEL[PLURAL]} to extract: {len(qids)}") item_count = 0 extract_dicts = [] chunk_size = 50 # The chunksize 50 is allowed by the wikidata api, bigger numbers need special permissions id_chunks = chunks(list(qids), chunk_size) for chunk in id_chunks: query_result = wikidata_entity_request( chunk, props=[LABEL[PLURAL]], timeout=10) # country entities take longer so timeout is increased if ENTITIES not in query_result: logger.error("Skipping chunk") continue for result in query_result[ENTITIES].values(): try: qid = result[ID] except Exception as error: logger.error( "Error on qid, skipping item. Error: {0}".format(error)) continue label = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], EN, type_name) subject_dict = { ID: qid, LABEL[SINGULAR]: label, } for langkey in language_keys: label_lang = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], langkey, type_name) subject_dict.update( {f"{LABEL[SINGULAR]}_{langkey}": label_lang}) extract_dicts.append(subject_dict) item_count += len(chunk) print( f"Status of {type_name} {LABEL[PLURAL]}: {item_count}/{len(qids)}", end="\r", flush=True, ) print(datetime.datetime.now(), f"Finished with {type_name} {LABEL[PLURAL]}") return extract_dicts
def get_classes( type_name: str, qids: List[str], already_extracted_superclass_ids: Set[str] = set(), language_keys: Optional[List[str]] = lang_keys, ) -> List[Dict]: """Function to extract the classes of the extracted wikidata entities (meaning the 'instance of' attribute wikidata entity qids). Their subclasses are also extracted recursively (also called transitive closure) Args: type_name: oab type e. g. movement qids: List of qids to extract the labels from language_keys: All language keys which should be extracted. Defaults to languageconfig.csv already_extracted_superclass_ids: A list of already extracted superclass ids for the recursive calls, this is also the anchor to stop recursion Returns: Returns a list of dicts with the classes from the oab entities and their subclasses """ print(datetime.datetime.now(), f"Starting with {type_name}") if type_name == CLASS[PLURAL]: print( f"Total {type_name} to extract (only 'instance_of' of the provided qids): {len(qids)}" ) else: print( f"Total {type_name} to extract (only 'subclass_of' of the provided qids): {len(qids)}" ) item_count = 0 extract_dicts = [] chunk_size = 50 # The chunksize 50 is allowed by the wikidata api, bigger numbers need special permissions classes_id_chunks = chunks(list(qids), chunk_size) for chunk in classes_id_chunks: query_result = wikidata_entity_request(chunk) if ENTITIES not in query_result: logger.error("Skipping chunk") continue for result in query_result[ENTITIES].values(): try: qid = result[ID] except Exception as error: logger.error( "Error on qid, skipping item. Error: {0}".format(error)) continue label = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], EN, type_name) description = map_wd_attribute.try_get_label_or_description( result, DESCRIPTION[PLURAL], EN, type_name) subclass_of = map_wd_attribute.try_get_qid_reference_list( result, PROPERTY_NAME_TO_PROPERTY_ID[SUBCLASS_OF], type_name) class_dict = { ID: qid, LABEL[SINGULAR]: label, DESCRIPTION[SINGULAR]: description, SUBCLASS_OF: subclass_of, } for langkey in language_keys: label_lang = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], langkey, type_name) description_lang = map_wd_attribute.try_get_label_or_description( result, DESCRIPTION[PLURAL], langkey, type_name) class_dict.update({ f"{LABEL[SINGULAR]}_{langkey}": label_lang, f"{DESCRIPTION[SINGULAR]}_{langkey}": description_lang, }) extract_dicts.append(class_dict) item_count += len(chunk) print(f"Status of {type_name}: {item_count}/{len(qids)}", end="\r", flush=True) return load_entities_by_attribute_with_transitive_closure( extract_dicts, SUBCLASS_OF, CLASS[PLURAL], already_extracted_superclass_ids, get_classes, [], )
def extract_artworks( type_name: str, wikidata_id: str, already_crawled_wikidata_items: Set, dev_mode: bool, dev_chunk_limit: int, language_keys: Optional[List[str]] = lang_keys, ) -> List[Dict]: """Extracts artworks metadata from Wikidata and stores them in a dictionary. Args: type_name: Type name of an artwork e. g. 'drawings'. Important for console output wikidata_id: Wikidata Id of a class; all instances of this class and all subclasses with image will be loaded. See artworks_ids_query.sparql already_crawled_wikidata_items: Set of all already crawled artwork items. Because the types have common items it is necessary to avoid loading items multiple times language_keys: All language keys which should be extracted. Defaults to languageconfig.csv dev_mode: To reduce the number of loaded chunks set this to true dev_chunk_limit: Limit of chunks per category Returns: A list with all artwork entity dicts (or JSON-objects) which are transformed for the OAB Examples: extract_artworks('drawings', 'wd:Q93184', '('en', 'de')) extract_artworks('sculptures', 'wd:Q860861', '('en', 'de')) extract_artworks('paintings', 'wd:Q3305213', '('en', 'de')) """ print(datetime.datetime.now(), "Starting with", type_name) extract_dicts = [] chunk_count = 0 item_count = 0 artwork_ids = query_artwork_qids(type_name, wikidata_id) # Don't load items again, if they were loaded in another artwork category for artwork_id in artwork_ids: if artwork_id in already_crawled_wikidata_items: artwork_ids.remove(artwork_id) print( f"{len(artwork_ids)} {type_name} entries are not loaded yet, starting now. Already crawled item count is {len(already_crawled_wikidata_items)}" ) chunk_size = 50 # The chunksize 50 is allowed by the wikidata api, bigger numbers need special permissions artwork_id_chunks = chunks(artwork_ids, chunk_size) for chunk in artwork_id_chunks: if dev_mode and chunk_count == dev_chunk_limit: logger.info( f"DEV_CHUNK_LIMIT of {type_name} reached. End extraction for {type_name}" ) break query_result = wikidata_entity_request(chunk) if ENTITIES not in query_result: logger.error("Skipping chunk") continue for result in query_result[ENTITIES].values(): try: qid = result[ID] image = map_wd_attribute.get_image_url_by_name( result[CLAIMS][PROPERTY_NAME_TO_PROPERTY_ID[IMAGE]][0] [MAINSNAK][DATAVALUE][VALUE]) except Exception as error: logger.error( "Error on qid or image, skipping item. Qid: {0}, Image: {1}, Error: {2}" .format(qid, image, error)) continue label = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], EN, type_name) description = map_wd_attribute.try_get_label_or_description( result, DESCRIPTION[PLURAL], EN, type_name) ( classes, artists, locations, genres, movements, materials, motifs, main_subjects, exhibition_history, ) = map_wd_attribute.get_attribute_values_with_try_get_func( result, [ CLASS[SINGULAR], ARTIST[SINGULAR], LOCATION[SINGULAR], GENRE[SINGULAR], MOVEMENT[SINGULAR], MATERIAL[SINGULAR], MOTIF[SINGULAR], MAIN_SUBJECT[SINGULAR], EXHIBITION_HISTORY, ], type_name, map_wd_attribute.try_get_qid_reference_list, ) iconclasses = map_wd_attribute.try_get_value_list( result, PROPERTY_NAME_TO_PROPERTY_ID[ICONCLASS[SINGULAR]], type_name) inception = map_wd_attribute.try_get_year_from_property_timestamp( result, PROPERTY_NAME_TO_PROPERTY_ID[INCEPTION], type_name) country = map_wd_attribute.try_get_first_qid( result, PROPERTY_NAME_TO_PROPERTY_ID[COUNTRY], type_name) # Resolve dimensions # The units are qids which have to be resolved later ( height, width, length, diameter, ) = map_wd_attribute.get_attribute_values_with_try_get_func( result, [HEIGHT, WIDTH, LENGTH, DIAMETER], type_name, map_wd_attribute.try_get_dimension_value, ) ( height_unit, width_unit, length_unit, diameter_unit, ) = map_wd_attribute.get_attribute_values_with_try_get_func( result, [HEIGHT, WIDTH, LENGTH, DIAMETER], type_name, map_wd_attribute.try_get_dimension_unit, ) significant_events = map_wd_attribute.try_get_significant_events( result) artwork_dictionary = { ID: qid, CLASS[PLURAL]: classes, LABEL[SINGULAR]: label, DESCRIPTION[SINGULAR]: description, IMAGE: image, ARTIST[PLURAL]: artists, LOCATION[PLURAL]: locations, GENRE[PLURAL]: genres, MOVEMENT[PLURAL]: movements, INCEPTION: inception, MATERIAL[PLURAL]: materials, MOTIF[PLURAL]: motifs, COUNTRY: country, HEIGHT: height, HEIGHT_UNIT: height_unit, WIDTH: width, WIDTH_UNIT: width_unit, LENGTH: length, LENGTH_UNIT: length_unit, DIAMETER: diameter, DIAMETER_UNIT: diameter_unit, ICONCLASS[PLURAL]: iconclasses, MAIN_SUBJECT[PLURAL]: main_subjects, EXHIBITION_HISTORY: exhibition_history, SIGNIFICANT_EVENT: significant_events, TYPE: ARTWORK[SINGULAR], } # Apply blocklist to artwork dictionary for t in [ CLASS[PLURAL], ARTIST[PLURAL], LOCATION[PLURAL], GENRE[PLURAL], MOVEMENT[PLURAL], MATERIAL[PLURAL], MOTIF[PLURAL], ICONCLASS[PLURAL], MAIN_SUBJECT[PLURAL], EXHIBITION_HISTORY ]: try: artwork_dictionary[t] = list( set(artwork_dictionary[t]) - set(BLOCKLIST)) except Exception as e: logger.exception(e) continue for langkey in language_keys: label_lang = map_wd_attribute.try_get_label_or_description( result, LABEL[PLURAL], langkey, type_name) description_lang = map_wd_attribute.try_get_label_or_description( result, DESCRIPTION[PLURAL], langkey, type_name) wikipedia_link_lang = map_wd_attribute.try_get_wikipedia_link( result, langkey, type_name) artwork_dictionary.update({ f"{LABEL[SINGULAR]}_{langkey}": label_lang, f"{DESCRIPTION[SINGULAR]}_{langkey}": description_lang, f"{WIKIPEDIA_LINK}_{langkey}": wikipedia_link_lang, }) extract_dicts.append(artwork_dictionary) already_crawled_wikidata_items.add(qid) item_count += len(chunk) print( f"Status of {type_name}: {item_count}/{len(artwork_ids)}", end="\r", flush=True, ) chunk_count += 1 print(datetime.datetime.now(), "Finished with", type_name) return extract_dicts