class ImporterBot(object): """Bot to enrich/create info on Wikidata for Australian heritage items.""" def __init__(self, base_path, new=False, cutoff=None, preview_file=None): """ Initialise the ImporterBot. :param base_path: path to the output directory :param new: whether to also create new items :param cutoff: the number of items to process before stopping. None being interpreted as all. :param preview_file: run in demo mode (create previews rather than live edits) and output the result to this file. """ self.repo = pywikibot.Site().data_repository() self.wd = WdS(self.repo, EDIT_SUMMARY) self.new = new self.cutoff = cutoff if preview_file: self.demo = True self.preview_file = path.join(base_path, preview_file) else: self.demo = False self.preview_data = [] self.set_references() self.place_id_p = 'P3008' # unique identifier property self.country = self.wd.QtoItemPage('Q408') self.states = self.make_states_map() self.settlements = self.make_settlements_map() self.hectares = self.wd.QtoItemPage(helpers.get_unit_q('ha')) self.make_status_and_instance_map() self.place_id_items = helpers.fill_cache_wdqs(self.place_id_p, no_strip=True) def set_references(self): """Set the three types of references needed.""" self.ref = { 'national': self.make_url_ref( 'http://data.gov.au/dataset/2016-soe-her-aus-national-heritage', # noqa '2017-07-21', '2017-06-07'), 'commonwealth': self.make_url_ref( 'http://data.gov.au/dataset/commonwealth-heritage-list', '2017-07-21', '2017-05-31') } self.coord_ref = { 'national': self.make_url_ref( 'http://www.environment.gov.au/heritage/places/national-heritage-list', # noqa '2017-08-13'), 'commonwealth': self.make_url_ref( 'https://data.gov.au/dataset/57720684-4948-45db-a2c8-37259d531d87', # noqa '2017-08-13', '2017-07-10') } def make_status_and_instance_map(self): """Construct mapping for cultural heritage status and instance type.""" self.status = { 'national': self.wd.QtoItemPage('Q20747146'), 'commonwealth': self.wd.QtoItemPage('Q30108476') } self.instance_type = { 'indigenous': self.wd.QtoItemPage('Q38048771'), 'historic': self.wd.QtoItemPage('Q38048707'), 'natural': self.wd.QtoItemPage('Q38048753') } def make_settlements_map(self): """Retrieve Australian settlements with state/territory connection.""" sparql = ( "SELECT DISTINCT ?city ?cityLabel ?admin ?adminLabel " "WHERE " "{ " "?city wdt:P31/wdt:P279* wd:Q486972 . " "?city wdt:P17 wd:Q408 . " "?city wdt:P131* ?admin . " "{ ?admin wdt:P31 wd:Q5852411 . }" "UNION" "{ ?admin wdt:P31 wd:Q14192252 . }" "UNION" "{ ?admin wdt:P31 wd:Q14192199 . }" 'SERVICE wikibase:label { bd:serviceParam wikibase:language "en" . }' # noqa "}") data = wdqs.make_simple_wdqs_query(sparql) settlements = dict() for d in data: state_qid = d['admin'].split('/')[-1] city_qid = d['city'].split('/')[-1] city_name = d['cityLabel'] if city_name not in settlements: settlements[city_name] = [] settlements[city_name].append({ 'state': state_qid, 'qid': city_qid }) return settlements def make_states_map(self): """ Retrieve the state/territory mappings from Wikidata. Also tries to match items for the EXT and OS codes. """ sparql = ("SELECT ?item ?iso " "WHERE " "{ " "?item wdt:P300 ?value . " "?item wdt:P17 wd:Q408 . " "BIND(REPLACE(?value, 'AU-', '', 'i') AS ?iso) " "}") data = wdqs.make_select_wdqs_query(sparql, 'item', 'iso') states = dict() for k, v in data.items(): states[v] = self.wd.QtoItemPage(k) # external territories (random hits mapped) states['EXT'] = { 'Ashmore and Cartier Islands': self.wd.QtoItemPage('Q133888'), "Australian Antarctic Territory|Dumont D'Urville Station|Mawson Station": self.wd.QtoItemPage('Q178994'), # noqa 'Christmas Island|Settlement|Drumsite|Poon Saan': self.wd.QtoItemPage('Q31063'), # noqa 'Cocos (Keeling) Islands': self.wd.QtoItemPage('Q36004'), 'Coral Sea Islands': self.wd.QtoItemPage('Q172216'), 'Heard and McDonald Islands': self.wd.QtoItemPage('Q131198'), 'Jervis Bay Territory': self.wd.QtoItemPage('Q15577'), 'Norfolk Island|Kingston|Longridge|Burnt Pine|Middlegate': self.wd.QtoItemPage('Q31057') # noqa } # OS other state? states['OS'] = { 'United Kingdom': self.wd.QtoItemPage('Q145'), 'USA': self.wd.QtoItemPage('Q30') } return states def make_url_ref(self, url, fetch_date, publish_date=None): """Make a Reference object for a url. Contains 3 parts: * P813: Retrieval date * P577: Publication date <from creation date of the document> * P854: Reference url <using the input url> :param url: the source url :param fetch_date: the retrieval date url (iso) :param publish_date: the retrieval date url (iso) :return: WdS.Reference """ date_claims = [] if publish_date: date_claims.append( self.wd.make_simple_claim('P577', helpers.iso_to_WbTime(publish_date))) date_claims.append( self.wd.make_simple_claim('P813', helpers.iso_to_WbTime(fetch_date))) ref = WdS.Reference( source_test=[self.wd.make_simple_claim('P854', url)], source_notest=date_claims) return ref def output_previews(self): """Output any PreviewItems to the preview_file.""" with open(self.preview_file, 'w', encoding='utf-8') as f: for preview in self.preview_data: f.write(preview.make_preview_page()) f.write('--------------\n\n') pywikibot.output('Created "{}" for previews'.format(self.preview_file)) def process_all_objects(self, data): """ Handle all the Australian heritage objects. Only increments counter when an object is updated. :param data: dict of all the heritage objects. """ count = 0 for place_id, entry_data in data.items(): if self.cutoff and count >= self.cutoff: break item = None if place_id in self.place_id_items: item = self.wd.QtoItemPage(self.place_id_items[place_id]) if item or self.new: self.process_single_object(entry_data, item) count += 1 def process_single_object(self, data, item): """ Process a single Australian heritage object. :param data: dict of data for a single object :param item: Wikidata item associated with an object, or None if one should be created. """ if not self.demo: item = item or self.create_new_place_id_item(data) item.exists() # load the item contents # Determine claims labels = self.make_labels(data) descriptions = self.make_descriptions(data) protoclaims = self.make_protoclaims(data) ref = self.ref[self.get_heritage_type(data['type'])] # Upload claims if self.demo: self.preview_data.append( PreviewItem(labels, descriptions, protoclaims, item, ref)) else: self.commit_labels(labels, item) self.commit_descriptions(descriptions, item) self.commit_claims(protoclaims, item, ref) def create_new_place_id_item(self, data): """ Create a new place_id item with some basic info and return it. :param data: dict of data for a single object :return: pywikibot.ItemPage """ labels = helpers.convert_language_dict_to_json(self.make_labels(data), typ='labels') desc = helpers.convert_language_dict_to_json( self.make_descriptions(data), typ='descriptions') id_claim = self.wd.make_simple_claim(self.place_id_p, data.get('place_id')) item_data = { "labels": labels, "descriptions": desc, "claims": [ id_claim.toJSON(), ] } try: return self.wd.make_new_item(item_data, EDIT_SUMMARY) except pywikibot.data.api.APIError as e: raise pywikibot.Error('Error during item creation: {:s}'.format(e)) def make_labels(self, data): """ Make a label object from the available info. :param data: dict of data for a single object :return: label dict """ labels = {} name = data.get('name') if name: labels['en'] = [ name.replace(' ', ' ').strip(), ] return labels def make_descriptions(self, data): """ Make a description object in English. Address is partitioned so as to include the place name and territory/state in case these are not included anywhere later. :param data: dict of data for a single object :return: description object """ text = '{heritage_type} {list_type} heritage site in {address}' descriptions = { 'en': text.format(heritage_type=data['class'].lower(), list_type=self.get_heritage_type(data['type']), address=data['address'].rpartition(',')[2].strip()) } return descriptions def commit_labels(self, labels, item): """ Add labels and aliases to item. :param labels: label object :param item: item to add labels to """ if labels: self.wd.add_multiple_label_or_alias(labels, item, case_sensitive=False) def commit_descriptions(self, descriptions, item): """ Add descriptions to item. :param descriptions: description object :param item: item to add descriptions to """ if descriptions: self.wd.add_multiple_descriptions(descriptions, item) def commit_claims(self, protoclaims, item, default_ref): """ Add each claim (if new) and source it. :param protoclaims: a dict of claims with key: Prop number val: Statement|list of Statements :param item: the target entity :param default_ref: main/default reference to use """ for prop, statements in protoclaims.items(): if statements: statements = helpers.listify(statements) statements = set(statements) # eliminate potential duplicates for statement in statements: # check if None or a Statement(None) if (statement is not None) and (not statement.isNone()): # use internal reference if present, else the general ref = statement.ref or default_ref self.wd.addNewClaim(prop, statement, item, ref) # reload item so that next call is aware of changes item = self.wd.QtoItemPage(item.title()) item.exists() def make_protoclaims(self, data): """ Construct potential claims for an entry. :param data: dict of data for a single heritage object """ protoclaims = dict() # P17: country protoclaims['P17'] = WdS.Statement(self.country) # P1435: heritage status heritage_type = self.get_heritage_type(data.get('type')) statement = WdS.Statement(self.status[heritage_type]) if data.get('register_date'): statement.addQualifier( WdS.Qualifier('P580', self.parse_date(data.get('register_date')))) protoclaims['P1435'] = statement # P31: class protoclaims['P31'] = WdS.Statement( self.instance_type[data.get('class').lower()]) # P3008: place_id protoclaims[self.place_id_p] = WdS.Statement(data['place_id']) # P131: state protoclaims['P131'] = WdS.Statement( self.get_state(data['state'], data['address'])) # P2046: area if data.get('hectares'): protoclaims['P2046'] = WdS.Statement( pywikibot.WbQuantity(data['hectares'], unit=self.hectares, site=self.wd.repo)) # P969: address if ',' in data['address']: protoclaims['P969'] = WdS.Statement(data['address']) # P276: place protoclaims['P276'] = WdS.Statement( self.get_place(data['state'], data['address'])) # P625: coordinate if data.get('lat') and data.get('lon'): protoclaims['P625'] = self.get_coordinate_statement( data.get('lat'), data.get('lon'), heritage_type) return protoclaims def get_coordinate_statement(self, lat, lon, heritage_type): """Construct a Statement for the provided coordinates.""" statement = WdS.Statement( pywikibot.Coordinate(float(lat), float(lon), globe='earth', precision=DEFAULT_PREC)) statement.add_reference(self.coord_ref[heritage_type]) return statement def get_heritage_type(self, typ): """Determine which heritage type the object is.""" heritage_type = None if typ.startswith('Q1116950'): heritage_type = 'commonwealth' elif typ.startswith('Q781601'): heritage_type = 'national' else: pywikibot.error('Unrecognized status: {0}'.format(typ)) return heritage_type def parse_date(self, date): """Convert date in DD-MMM-YYYY format to WbTime.""" months = [ 'JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC' ] dd, mmm, yyyy = date.split('-') iso = '{year}-{month:02d}-{day:02d}'.format(year=yyyy, day=int(dd), month=months.index(mmm) + 1) return helpers.iso_to_WbTime(iso) def get_place(self, state, address): """ Determine which settlement the object is in. The format of address is "street, place STATE_ISO" """ place = address.rpartition(',')[2][:-len(state)].strip() state_item = self.get_state(state, address) if place in self.settlements and state_item: hits = [] for candidate in self.settlements[place]: if candidate['state'] == state_item.id: hits.append(candidate['qid']) if len(set(hits)) == 1: return self.wd.QtoItemPage(hits[0]) def get_state(self, state, address): """Determine which state/territory the object is in.""" state_item = None if state not in self.states: pywikibot.error('Unrecognized state: {0}'.format(state)) elif state == 'EXT': address = address[:-len('EXT')].strip() for key, v in self.states['EXT'].items(): if any(address.endswith(k) for k in key.split('|')): state_item = v break elif state == 'OS': for k, v in self.states['OS'].items(): if address.endswith(k): state_item = v break else: state_item = self.states[state] return state_item
class PaintingsBot: """Bot to enrich, and create, for items about paintings on Wikidata.""" def __init__(self, dict_generator, painting_id_prop, cache_max_age=0): """Initiate the bot, loading files and querying WDQ. @param dict_generator: The generator for the Europeana painting objects @type dict_generator: generator (that yields Dict objects). @param painting_id_prop: the P-id of the painting-id property @type painting_id_prop: str @param cache_max_age: Max age of local wdq cache, defaults to 0 @type cache_max_age: int """ self.generator = dict_generator self.repo = pywikibot.Site().data_repository() self.commons = pywikibot.Site(u'commons', u'commons') self.wd = WD(self.repo) self.add_new = False # If new objects should be created self.skip_miniatures = True # If (new) miniatures should be skipped # Load prefixes and find allowed collections collections = set([INSTITUTION_Q]) self.mappings = helpers.load_json_file('mappings.json', force_path=__file__) self.prefix_map = self.mappings['prefix_map'] self.bad_prefix = self.mappings['bad_prefix'] for p, k in self.prefix_map.iteritems(): if k['subcol'] is not None: collections.add(k['subcol'].strip('Q')) self.collections = list(collections) # Set log file self.log = codecs.open(u'nationalmuseumSE.log', 'a', 'utf-8') # Load creator dump file self.creator_dump = helpers.load_json_file('Oku_NM_arbetskopia.json', force_path=__file__) # hard-coded anons e.g. "unknown swedish 17th century" anons = helpers.load_json_file('anons.json', force_path=__file__) # prepare WDQ painting query query = u'CLAIM[195:%s] AND CLAIM[%s]' % \ (',195:'.join(self.collections), painting_id_prop) self.painting_ids = helpers.fill_cache(painting_id_prop, queryoverride=query, cache_max_age=cache_max_age) # prepare WDQ artist query (nat_mus_id - Q_id pairs) self.artist_ids = helpers.fill_cache('P2538', cache_max_age=cache_max_age) # add anons for a in anons: self.artist_ids[a] = ANON_Q self.painting_id_prop = 'P%s' % painting_id_prop def run(self): """Start the robot.""" self.creators = {} for painting in self.generator: # isolate ids ids = painting['object']['proxies'][0]['dcIdentifier']['def'] painting_id = ids[0].replace('Inv Nr.:', '').strip('( )') obj_id = ids[1] # Museum contains several sub-collections. Only handle mapped ones if painting_id.split(' ')[0] in self.prefix_map.keys(): self.process_painting(painting, painting_id, obj_id) elif painting_id.split(' ')[0] not in self.bad_prefix: pywikibot.output(u'Skipped due to unknown collection: %s' % painting_id) def process_painting(self, painting, painting_id, obj_id): """Process a single painting. This will also create it if self.add_new is True. @param painting: information object for the painting @type painting: dict @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @param obj_id: the internal id of the painting in the Nationalmuseum database. @type obj_id: str """ uri = u'http://collection.nationalmuseum.se/eMuseumPlus?service=' \ u'ExternalInterface&module=collection&objectId=%s&viewType=' \ u'detailView' % obj_id europeana_url = u'http://europeana.eu/portal/record%s.html' % \ painting['object']['about'] painting_item = None # newclaims = [] if painting_id in self.painting_ids: painting_item = self.create_existing_painting( painting, painting_id) elif self.add_new and not (self.skip_miniatures and PaintingsBot.is_miniature(painting)): # if objection collection is allowed and # unless it is a miniature and we are skipping those painting_item = self.create_new_painting(painting, painting_id, europeana_url, uri) # add new claims if painting_item and painting_item.exists(): data = painting_item.get(force=True) claims = data.get('claims') # add natmus id claim self.add_natmus_id(painting_item, obj_id, uri) # add inventory number with collection self.add_inventory_and_collection_claim(painting_item, painting_id, painting, uri) # Instance_of if u'P31' not in claims: self.add_instanceof_claim(painting_item, painting_id, painting) # title (as claim) # commented out as the titles in Europeana are not reliable # if u'P1476' not in claims: # self.add_title_claim(painting_item, painting) # Europeana_ID self.add_europeana_claim(painting_item, painting) # Check for potential images to add, if none is present if u'P18' not in claims: self.add_image_claim(painting_item, uri) # creator through Nat_mus_database dump self.add_natmus_creators(painting_item, obj_id, uri) # creator IFF through dbpedia # if u'P170' not in claims: # self.add_dbpedia_creator(painting_item, painting) def add_title_claim(self, painting_item, painting): """Add a title/P1476 claim based on dcTitle. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting: information object for the painting @type painting: dict """ dc_title = painting['object']['proxies'][0]['dcTitle'] titles = [] for lang, title in dc_title.iteritems(): titles.append(pywikibot.WbMonolingualText(title[0], lang)) for title in titles: self.wd.addNewClaim(u'P1476', WD.Statement(title), painting_item, self.make_europeana_reference(painting)) def add_locatedin_claim(self, painting_item, painting_id, painting): """Add a located_in/P276 claim based on sub-collection. No longer used as sub-collection does not match actual placing. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @param painting: information object for the painting @type painting: dict """ place = self.prefix_map[painting_id.split(' ')[0]]['place'] place_item = self.wd.QtoItemPage(place) self.wd.addNewClaim(u'P276', WD.Statement(place_item), painting_item, self.make_europeana_reference(painting)) def add_dbpedia_creator(self, painting_item, painting): """Add a Creator/P170 claim through a dbpedia look-up. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting: information object for the painting @type painting: dict """ creator_id = None try: db_creator = painting['object']['proxies'][1]['dcCreator']['def'] if len(db_creator) == 1: # skip anything more complex than one creator db_creator = db_creator[0].strip() if db_creator.startswith('http://dbpedia.org/resource/'): if db_creator not in self.creators.keys(): self.creators[db_creator] = \ helpers.dbpedia_2_wikidata(db_creator) creator_id = self.creators[db_creator] except KeyError: return if creator_id: self.set_creator(painting_item, self.make_europeana_reference(painting), creator_q=creator_id) def add_image_claim(self, painting_item, uri): """Add a image/P18 claim if exactly one image is found on Commons. Uses the nationalmuseum.se uri to search for matches on Commons. Adds a claim only if a unique hit is found. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param uri: reference url on nationalmuseum.se @type uri: str """ images = self.file_from_external_link(uri) if len(images) > 1: # for now don't want to choose the appropriate one pywikibot.output('Found multiple matching images for %s' % painting_item) for image in images: pywikibot.output(u'\t%s' % image) elif len(images) == 1: self.wd.addNewClaim(u'P18', WD.Statement(images[0]), painting_item, self.make_commons_reference()) def add_europeana_claim(self, painting_item, painting): """Add a Europeana ID/P727 claim. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting: information object for the painting @type painting: dict """ europeana_prop = u'P727' europeana_id = painting['object']['about'].lstrip('/') # abort if conflicting info if europeana_prop in painting_item.claims and \ not self.wd.has_claim(europeana_prop, europeana_id, painting_item): pywikibot.output(u'%s has conflicting %s. Expected %s' % (painting_item, europeana_prop, europeana_id)) return self.wd.addNewClaim(europeana_prop, WD.Statement(europeana_id), painting_item, self.make_europeana_reference(painting)) def add_instanceof_claim(self, painting_item, painting_id, painting): """Add an instance_of/P31 claim. Instance_of is always painting or icon while working on the paintings collection. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @param painting: information object for the painting @type painting: dict """ dcformat_item = self.wd.QtoItemPage(PAINTING_Q) # painting if painting_id.split(' ')[0] == 'NMI': dcformat_item = self.wd.QtoItemPage(ICON_Q) # icon self.wd.addNewClaim(u'P31', WD.Statement(dcformat_item), painting_item, self.make_europeana_reference(painting)) @staticmethod def is_miniature(painting): """Determine if the painting is a miniature. @param painting: information object for the painting @type painting: dict @rtype bool """ for concept in painting['object']['concepts']: if concept[u'about'] == MINIATURE_URL: # pywikibot.output(u'Skipping miniature') return True return False def create_existing_painting(self, painting, painting_id): """Add base info to an existing paining. Adds the same info as would have been added had it been created with create_new_painting() @param painting: information object for the painting @type painting: dict @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @return: the created painting item @rtype: pywikibot.ItemPage """ painting_item = self.wd.QtoItemPage(self.painting_ids.get(painting_id)) # check label data = painting_item.get() labels = make_labels(painting) new_labels = find_new_values(data, labels, 'labels') if new_labels: pywikibot.output('Adding label to %s' % painting_item.title()) painting_item.editLabels(new_labels) # check description descriptions = make_descriptions(painting) if descriptions: new_descr = find_new_values(data, descriptions, 'descriptions') if new_descr: pywikibot.output('Adding description to %s' % painting_item.title()) painting_item.editDescriptions(new_descr) return painting_item def create_new_painting(self, painting, painting_id, europeana_url, uri): """Create a new painting item and return it. @param painting: information object for the painting @type painting: dict @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @param europeana_url: reference url for Europeana @type europeana_url: str @param uri: reference uri at nationalmuseum.se @type uri: str @return: the created painting item @rtype: pywikibot.ItemPage """ data = {'labels': {}, 'descriptions': {}} data['labels'] = make_labels(painting) data['descriptions'] = make_descriptions(painting) if not data['descriptions']: return # print data # create new empty item and request Q-number summary = u'%s: Creating new item with data from %s' % (EDIT_SUMMARY, europeana_url) painting_item = None try: painting_item = self.wd.make_new_item(data, summary) except pywikibot.data.api.APIError as e: if e.code == u'modification-failed': # disambiguate and try again for lang, content in data['descriptions'].iteritems(): disambiguation = content['value'] + u' (%s)' % painting_id data['descriptions'][lang]['value'] = disambiguation try: painting_item = self.wd.make_new_item(data, summary) except pywikibot.data.api.APIError as e: if e.code == u'modification-failed': pywikibot.output(u'modification-failed error: ' u'skipping %s' % uri) return else: raise pywikibot.Error(u'Error during item creation: ' u'%s' % e) else: raise pywikibot.Error(u'Error during item creation: %s' % e) return painting_item def add_natmus_id(self, painting_item, obj_id, uri): """Add a natmus_painting_id/P2539 claim. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param obj_id: the nationalmuseum database id @type obj_id: str @param uri: reference url on nationalmuseum.se @type uri: str """ self.wd.addNewClaim(u'P2539', WD.Statement(obj_id), painting_item, self.make_url_reference(uri)) def add_natmus_creators(self, painting_item, obj_id, uri): """Add creator/P170 claim(s) based on the database dump info. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param obj_id: the nationalmuseum database id @type obj_id: str @param uri: reference url on nationalmuseum.se @type uri: str """ if obj_id not in self.creator_dump.keys(): return # each artwork may have multiple artists, # which must all be on wikidata for artist_id in self.creator_dump[obj_id].keys(): if artist_id not in self.artist_ids.keys(): self.logger('Artist not found on wikidata: %s' % artist_id) return dump_entry = self.creator_dump[obj_id] if len(dump_entry) == 1: artist_entry = dump_entry.iteritems().next() self.add_singel_natmus_creator(painting_item, artist_entry, uri) elif len(dump_entry) == 2: # self.add_double_natmus_creator(painting_item, dump_entry, uri) # skipping until duplication issue has been solved pass else: # for now avoid any entries with more creators return def add_singel_natmus_creator(self, painting_item, artist, uri): u"""Add a simple creator/P170 claim based on the database dump info. Handles cases with only a single identified creator. Either * Known creator * Unknown/uncertain creator somehow related to a known person where creator is someone whose function is in artist_labels. For Forgery/After work by the bot needs to be aware of both parties, and both must exist on Wikidata @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param artist: the dump entry for the artist @type artist: tuple (artist_id, artist_info) @param uri: reference url on nationalmuseum.se @type uri: str """ anonymous_combos = { u'Tillskriven': 'P1773', u'Hennes ateljé': 'P1774', u'Hans ateljé': 'P1774', u'Hennes skola': 'P1780', u'Hans skola': 'P1780', u'Hennes art': 'P1777', u'Hans art': 'P1777', } artist_labels = (u'Konstnär', u'Mästare', u'Utförd av') artist_id, artist_info = artist artist_q = self.artist_ids[artist_id] if artist_info.get('OkuBeschreibungS') or \ artist_info.get('OkuValidierungS'): # this always indicates some special case which we cannot handle # for now return if artist_info.get('OkuFunktionS') and \ artist_info.get('OkuFunktionS') in artist_labels: if len(artist_info.keys()) == 1: # i.e. all other are empty self.set_creator(painting_item, self.make_url_reference(uri), creator_q=artist_q) elif artist_info.get('OkuArtS') in anonymous_combos.keys() and \ len(artist_info.keys()) == 2: # anonymous but attributed to the artist related_info = { 'P': anonymous_combos[artist_info.get('OkuArtS')], 'itis': self.wd.QtoItemPage(artist_q) } self.set_creator(painting_item, self.make_url_reference(uri), related_info=related_info) elif not artist_info.get('OkuFunktionS') and artist_id == '1': # this is the special case of a completly unknown creator self.set_creator(painting_item, self.make_url_reference(uri)) def add_double_natmus_creator(self, painting_item, artists, uri): u"""Add a comlex creator/P170 claim based on the database dump info. Handles cases with two identified creators in a relation along the lines of "Painting/Forgery by X after a work by Y" The logic is: OkuFunktionS in derived_combos -> OkuKueID = creator of original OkuFunktionS in artist_labels -> OkuKueID = creator of derivative @param artists: the dump entries for the artists @type artists: dict of {artist_id: artist_info} @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param uri: reference url on nationalmuseum.se @type uri: str """ derived_combos = { u'Kopia efter': 'P1877', u'Efter': 'P1877', u'Förfalskning efter': 'P1778', } artist_labels = (u'Konstnär', u'Utförd av') # set up targets original = None derivative = None relation = None for artist in artists.iteritems(): artist_id, artist_info = artist if artist_info.get('OkuBeschreibungS') or \ artist_info.get('OkuValidierungS'): # this indicates some special case which we cannot handle # for now return if artist_info.get('OkuFunktionS') and \ len(artist_info.keys()) == 1: # cannot deal with OkuArtS if artist_info.get('OkuFunktionS') in artist_labels: derivative = artist elif artist_info.get('OkuFunktionS') in derived_combos.keys(): original = artist relation = derived_combos[artist_info.get('OkuFunktionS')] # verify that both roles were filled if any(creator is None for creator in (original, derivative)): return # construct info and set original_q = self.artist_ids[original[0]] derivative_q = self.artist_ids[derivative[0]] related_info = {'P': relation, 'itis': self.wd.QtoItemPage(original_q)} self.set_creator(painting_item, self.make_url_reference(uri), creator_q=derivative_q, related_info=related_info) def set_creator(self, target_item, reference, creator_q=None, related_info=None): """Set a creator/P170 claim for a creator or creator combo. Allows for simple claims as well as more complex "in the manner of" etc. @param target_item: item to which claim is added @type target_item: pywikibot.ItemPage @param reference: the reference for the statment @type reference: WD.Reference @param related_info: related info as a dict with P/itis pairs @type related_info: dict @param creator_q: the Q-id of the creator @type creator_q: str """ creator_q = creator_q or ANON_Q creator_statement = WD.Statement(self.wd.QtoItemPage(creator_q)) # set any related qualifiers if related_info: creator_statement.addQualifier( WD.Qualifier(P=related_info['P'], itis=related_info['itis'])) # set claim self.wd.addNewClaim(u'P170', creator_statement, target_item, reference) def add_inventory_and_collection_claim(self, painting_item, painting_id, painting, uri): """Add an inventory_no, with qualifier, and a collection/P195 claim. This will add the collection qualifier to any matching claim missing it. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @param painting: information object for the painting @type painting: dict @param uri: reference url on nationalmuseum.se @type uri: str """ nationalmuseum_item = self.wd.QtoItemPage(INSTITUTION_Q) collection_p = u'P195' # abort if conflicting info if self.painting_id_prop in painting_item.claims and \ not self.wd.has_claim(self.painting_id_prop, painting_id, painting_item): pywikibot.output( u'%s has conflicting inv. no (%s). Expected %s' % (painting_item, self.painting_id_prop, painting_id)) return # add inventory number with collection self.wd.addNewClaim( self.painting_id_prop, WD.Statement(painting_id).addQualifier(WD.Qualifier( P=collection_p, itis=nationalmuseum_item), force=True), painting_item, self.make_url_reference(uri)) # add collection (or subcollection) subcol = self.prefix_map[painting_id.split(' ')[0]]['subcol'] collection_item = nationalmuseum_item if subcol is not None: collection_item = self.wd.QtoItemPage(subcol) self.wd.addNewClaim(collection_p, WD.Statement(collection_item), painting_item, self.make_europeana_reference(painting)) def make_europeana_reference(self, painting): """Make a Reference object with a Europeana retrieval url and today's date. @param uri: retrieval uri/url @type uri: str @rtype: WD.Reference """ europeana_url = u'http://europeana.eu/portal/record%s.html' % \ painting['object']['about'] return self.make_url_reference(europeana_url) def make_url_reference(self, uri): """Make a Reference object with a retrieval url and today's date. @param uri: retrieval uri/url @type uri: str @rtype: WD.Reference """ date = helpers.today_as_WbTime() ref = WD.Reference(source_test=self.wd.make_simple_claim(u'P854', uri), source_notest=self.wd.make_simple_claim( u'P813', date)) return ref def make_commons_reference(self): """Make a Reference object saying imported from Wikimedia Commons.""" commons_item = self.wd.QtoItemPage(COMMONS_Q) ref = WD.Reference(source_test=self.wd.make_simple_claim( u'P143', commons_item)) # imported from return ref def file_from_external_link(self, uri): """Identify files from a Nationalmuseum uri. Hits are any files containing a link to the eMuseumPlus uri. @param uri: reference url on nationalmuseum.se @type uri: str @return: matching images @rtype: list """ images = [] uri = uri.split('://')[1] objgen = pagegenerators.LinksearchPageGenerator(uri, namespaces=[6], site=self.commons) for page in objgen: images.append(pywikibot.FilePage(self.commons, page.title())) # I have no clue how the above results in duplicates, but it does so... images = list(set(images)) return images def most_missed_creators(self, cache_max_age=0): """Produce list of most frequent, but unlinked, creators. Query WDQ for all objects in the collection missing an artist then put together a top-list for most desired creator """ expected_items = [] query = u'CLAIM[195:%s] AND NOCLAIM[170]' % \ ',195:'.join(self.collections) # collection wd_queryset = wdquery.QuerySet(query) wd_query = wdquery.WikidataQuery(cacheMaxAge=cache_max_age) data = wd_query.query(wd_queryset) if data.get('status').get('error') == 'OK': expected_items = data.get('items') creator_dict = {} counter = 0 for q_val in expected_items: q_item = self.wd.QtoItemPage(q_val) data = q_item.get() claims = data.get('claims') if u'P170' in claims: continue descr = data.get('descriptions').get('en') if descr and descr.startswith(u'painting by '): creator = descr[len(u'painting by '):] if '(' in creator: # to get rid of disambiguation addition creator = creator[:creator.find('(')].strip() if creator in creator_dict.keys(): creator_dict[creator] += 1 else: creator_dict[creator] = 1 counter += 1 pywikibot.output(u'Found %d mentions of %d creators' % (counter, len(creator_dict))) # output f = codecs.open(u'creatorHitlist.csv', 'w', 'utf-8') for k, v in creator_dict.iteritems(): f.write(u'%d|%s\n' % (v, k)) f.close() def logger(self, text): """Append text to logfile. @param text: text to output @type text: str """ self.log.write(u'%s\n' % text) self.log.flush() # because shit tends to crash