class KulturnavBot(object): """Bot to enrich and create information on Wikidata from KulturNav info.""" EDIT_SUMMARY = 'import using #Kulturnav data' KULTURNAV_ID_P = '1248' GEONAMES_ID_P = '1566' SWE_KOMMUNKOD_P = '525' SWE_COUNTYKOD_P = '507' PLACE_P = '276' TIME_P = '585' # date DATASET_Q = None DISAMBIG_Q = '4167410' IS_A_P = '31' CATALOG_P = '972' DATASET_ID = None ENTITY_TYPE = None MAP_TAG = None COUNTRIES = [] # a list of country Q's ADMIN_UNITS = [] # a list of municipality+county Q's locations = {} # a dict of uuid to wikidata location matches current_uuid = '' # for debugging def __init__(self, dictGenerator, cache_max_age, verbose=False): """ Initialise the bot. Arguments: * generator - A generator that yields Dict objects. """ self.generator = dictGenerator self.repo = pywikibot.Site().data_repository() self.cutoff = None self.verbose = verbose self.require_wikidata = True self.cache_max_age = cache_max_age # trigger wdq query self.itemIds = helpers.fill_cache(self.KULTURNAV_ID_P, cache_max_age=cache_max_age) # set up WikidataStuff instance self.wd = WD(self.repo, self.EDIT_SUMMARY) # load lists self.COUNTRIES = wdqsLookup.wdq_to_wdqs(u'TREE[6256][][31]') self.ADMIN_UNITS = wdqsLookup.wdq_to_wdqs(u'TREE[15284][][31]') @classmethod def set_variables(cls, dataset_q=None, dataset_id=None, entity_type=None, map_tag=None, edit_summary=None): """Override any class variables. Used when command line arguments affect which type of run to do. @param dataset_q: the Q-id of the dataset @type dataset_q: str @param dataset_id: the uuid of the dataset @type dataset_id: str @param entity_type: the entity type to provide for the search API @type entity_type: str @param map_tag: the map_tag to use in the search API to find wikidata matches @type map_tag: str @param edit_summary: the edit_summary to use @type edit_summary: str """ cls.DATASET_Q = dataset_q or cls.DATASET_Q cls.DATASET_ID = dataset_id or cls.DATASET_ID cls.ENTITY_TYPE = entity_type or cls.ENTITY_TYPE cls.MAP_TAG = map_tag or cls.MAP_TAG cls.EDIT_SUMMARY = edit_summary or cls.EDIT_SUMMARY def run(self): """Start the robot.""" raise NotImplementedError("run() is not implemented in the base bot.") def runLayout(self, datasetRules, datasetProtoclaims, datasetSanityTest, label, shuffle): """ Execute the basic layout of a run. It should be called for a dataset-specific run which sets the parameters. param datasetRules: a dict of additional Rules or values to look for param datasetProtoclaims: a function for populating protoclaims param datasetSanityTest: a function which must return true for results to be written to Wikidata param label: the key in values to be used for label/alias. set to None to skip addNames() param shuffle: whether name/label/alias is shuffled or not i.e. if name = last, first """ count = 0 for hit in self.generator: # print count, self.cutoff if self.cutoff and count >= self.cutoff: break # some type of feedback if count % 100 == 0 and count > 0: pywikibot.output('%d entries handled...' % count) # Required rules/values to search for rules = { u'identifier': None, u'modified': None, u'seeAlso': None, u'sameAs': None, u'exactMatch': None, # not expected u'wikidata': None, u'libris-id': None, u'viaf-id': None, u'getty_aat': None, u'ulan': None } rules.update(datasetRules) # put together empty dict of values then populate values = {} for k in rules.keys(): values[k] = None if not self.populateValues(values, rules, hit): # continue with next hit if problem was encounterd continue # find the matching wikidata item hitItem = self.wikidataMatch(values) self.current_uuid = values['identifier'] # @todo: self.current_protoclaims # allows these to be accessed more easily # convert values to potential claims protoclaims = datasetProtoclaims(self, values) self.make_base_protoclaims(values, protoclaims) # output info for testing if self.verbose: pywikibot.output(values) pywikibot.output(protoclaims) pywikibot.output(hitItem) # Add information if a match was found if hitItem and hitItem.exists(): # if redirect then get target instead # make sure it passes the sanityTests if not self.sanityTest(hitItem): continue if not datasetSanityTest(self, hitItem): continue # add name as label/alias if label is not None: self.addNames(values[label], hitItem, shuffle=shuffle) # get the "last modified" timestamp and construct a Reference date = helpers.iso_to_WbTime(values[u'modified']) ref = self.make_ref(date) # add each property (if new) and source it self.addProperties(protoclaims, hitItem, ref) # allow for limited runs count += 1 # done pywikibot.output(u'Handled %d entries' % count) def populateValues(self, values, rules, hit): """ Populate values and check results given a hit. Given a list of values and a kulturnav hit, populate the values and check if result is problem free. @todo: raise Error instead of using problemFree solution param values: dict with keys and every value as None param rules: a dict with keys and values either: None: the exakt key is present in hit and its value is wanted a Rule: acording to the class above param hit: a kulturnav entry return bool problemFree """ ids = {} problemFree = True for entries in hit[u'@graph']: # populate ids for viaId rules if '@id' in entries.keys(): if entries['@id'] in ids.keys(): pywikibot.output('Non-unique viaID key: \n%s\n%s' % (entries, ids[entries['@id']])) ids[entries['@id']] = entries for entries in hit[u'@graph']: # handle rules for key, rule in rules.iteritems(): val = None if rule is None: if key in entries.keys(): val = entries[key] elif isinstance(rule, Rule): val = rule.resolve(entries, ids) # test and register found value if val is not None: if values[key] is None: values[key] = val else: pywikibot.output(u'duplicate entries for %s' % key) problemFree = False # the minimum which must have been identified if values[u'identifier'] is None: raise pywikibot.Error(u'Could not isolate the identifier from the ' u'KulturNav object! JSON layout must have ' u'changed. Crashing!') # dig into sameAs/exactMatch and seeAlso KulturnavBot.set_sameas_values(values) # only look at seeAlso if we found no Wikidata link and require one if self.require_wikidata and \ (not values[u'wikidata'] and values[u'seeAlso']): values[u'seeAlso'] = helpers.listify(values[u'seeAlso']) for sa in values[u'seeAlso']: if u'wikipedia' in sa: pywikibot.output(u'Found a Wikipedia link but no ' u'Wikidata link: %s %s' % (sa, values[u'identifier'])) problemFree = False if not problemFree: pywikibot.output(u'Found an issue with %s (%s), skipping' % (values['identifier'], values['wikidata'])) return problemFree def sanityTest(self, hitItem): """ Execute generic sanitytest which should be run independent on dataset. return bool """ return self.withoutClaimTest(hitItem, self.IS_A_P, self.DISAMBIG_Q, u'disambiguation page') def withoutClaimTest(self, hitItem, P, Q, descr): """ Execute base test that an item does not contain a particular claim. param hitItem: item to check param P: the property to look for param Q: the Q claim to look for param descr: a descriptive text return bool """ P = u'P%s' % P.lstrip('P') testItem = self.wd.QtoItemPage(Q) if self.wd.has_claim(P, testItem, hitItem): pywikibot.output(u'%s is matched to %s, ' u'FIXIT' % (hitItem.title(), descr)) return False else: return True def withClaimTest(self, hitItem, P, Q, descr, orNone=True): """ Execute base test that an item contains a certain claim. param hitItem: item to check param P: the property to look for param Q: (list) of Q claim to look for param descr: a descriptive text param orNone: if complete absence of the Property is also ok return bool """ P = u'P%s' % P.lstrip('P') Q = helpers.listify(Q) testItems = [] for q in Q: testItems.append(self.wd.QtoItemPage(q)) # check claims if P in hitItem.claims.keys(): for testItem in testItems: if self.wd.has_claim(P, testItem, hitItem): return True else: pywikibot.output(u'%s is identified as something other ' u'than a %s. Check!' % (hitItem.title(), descr)) return False elif orNone: # no P claim return True @staticmethod def set_sameas_values(values): """Isolate external identifiers through sameAs and exactMatch. @param values: All extracted values @type values: dict """ # merge sameAs and exactMatch match = helpers.bundle_values( [values[u'sameAs'], values[u'exactMatch']]) or [] # dig into sameAs/exactMatch and seeAlso for sa in match: if u'wikidata' in sa: values[u'wikidata'] = sa.split('/')[-1] elif u'libris-id' in values.keys() and \ u'libris.kb.se/auth/' in sa: values[u'libris-id'] = sa.split('/')[-1] elif u'viaf-id' in values.keys() and \ u'viaf.org/viaf/' in sa: values[u'viaf-id'] = sa.split('/')[-1] elif u'getty_aat' in values.keys() and \ u'vocab.getty.edu/aat/' in sa: values[u'getty_aat'] = sa.split('/')[-1] elif u'ulan' in values.keys() and \ u'vocab.getty.edu/ulan/' in sa: values[u'ulan'] = sa.split('/')[-1] def make_base_protoclaims(self, values, protoclaims): """Construct the protoclaims common for all KulturnavBots. Adds the claim to the protoclaims dict. @param values: the values extracted using the rules @type values: dict @param protoclaims: the dict of claims to add @type protoclaims: dict """ # kulturnav protoclaim incl. qualifier protoclaims[u'P%s' % self.KULTURNAV_ID_P] = \ WD.Statement(values[u'identifier']).addQualifier( WD.Qualifier( P=self.CATALOG_P, itis=self.wd.QtoItemPage(self.DATASET_Q)), force=True) # authority control protoclaims if values.get(u'libris-id'): protoclaims[u'P906'] = WD.Statement(values[u'libris-id']) if values.get(u'viaf-id'): protoclaims[u'P214'] = WD.Statement(values[u'viaf-id']) if values.get(u'getty_aat'): protoclaims[u'P1014'] = WD.Statement(values[u'getty_aat']) if values.get(u'ulan'): protoclaims[u'P245'] = WD.Statement(values[u'ulan']) def wikidataMatch(self, values): """ Find the matching wikidata item. Checks Wikidata first, then kulturNav. return ItemPage|None the matching item """ if values[u'identifier'] in self.itemIds: hitItemTitle = u'Q%s' % \ self.itemIds.get(values[u'identifier']) if not values[u'wikidata'] and not self.require_wikidata: # i.e. uuid has been supplied manually and exists on wikidata pass elif values[u'wikidata'] != hitItemTitle: # this may be caused by either being a redirect wd = self.wd.QtoItemPage(values[u'wikidata']) wi = self.wd.QtoItemPage(hitItemTitle) if wd.isRedirectPage() and wd.getRedirectTarget() == wi: pass elif wi.isRedirectPage() and wi.getRedirectTarget() == wd: pass else: pywikibot.output(u'Identifier missmatch (skipping): ' u'%s, %s, %s' % (values[u'identifier'], values[u'wikidata'], hitItemTitle)) return None elif values[u'wikidata']: hitItemTitle = values[u'wikidata'] else: # no match found return None # create ItemPage, bypassing any redirect hitItem = self.wd.bypassRedirect(self.wd.QtoItemPage(hitItemTitle)) # in case of redirect values[u'wikidata'] = hitItem.title() return hitItem def addNames(self, names, hitItem, shuffle=False): """ Prepare a nameObj or a list of such for add_label_or_alias(). param shuffle: bool if name order is last, first then this creates a local rearranged copy """ if names: if shuffle: namelist = [] if isinstance(names, dict): s = KulturnavBot.shuffle_names(names) if s is not None: namelist.append(s) elif isinstance(names, list): for n in names: s = KulturnavBot.shuffle_names(n) if s is not None: namelist.append(s) else: pywikibot.output(u'unexpectedly formatted name' u'object: %s' % names) if namelist: self.add_label_or_alias(namelist, hitItem) else: self.add_label_or_alias(names, hitItem) def addProperties(self, protoclaims, hitItem, ref): """ Add each property (if new) and source it. param protoclaims: a dict of claims with a key: Prop number val: Statement|list of Statments param hititem: the target entity param ref: WD.Reference """ for pcprop, pcvalue in protoclaims.iteritems(): if pcvalue: if isinstance(pcvalue, list): pcvalue = set(pcvalue) # eliminate potential duplicates for val in pcvalue: # check if None or a Statement(None) if (val is not None) and (not val.isNone()): self.wd.addNewClaim(pcprop, val, hitItem, ref) # reload item so that next call is aware of changes hitItem = self.wd.QtoItemPage(hitItem.title()) hitItem.exists() elif not pcvalue.isNone(): self.wd.addNewClaim(pcprop, pcvalue, hitItem, ref) # KulturNav specific functions def dbpedia2Wikidata(self, item): """ Convert dbpedia reference to the equivalent Wikidata item, if present. param item: dict with @language, @value keys return pywikibot.ItemPage|None """ if KulturnavBot.foobar(item): return if not all(x in item.keys() for x in (u'@value', u'@language')): pywikibot.output(u'invalid dbpedia entry: %s' % item) exit(1) # any site will work, this is just an example site = pywikibot.Site(item[u'@language'], 'wikipedia') page = pywikibot.Page(site, item[u'@value']) if page.properties().get(u'wikibase_item'): qNo = page.properties()[u'wikibase_item'] return self.wd.QtoItemPage(qNo) def db_gender(self, value): """Match gender values to items. Note that this returns a Statment unlike most other functions @param value: The gender value @type value: str @return: The gender item as a statement @rtype: WD.Statement or None """ known = { u'male': u'Q6581097', u'female': u'Q6581072', u'unknown': u'somevalue' } # a special case if value not in known.keys(): pywikibot.output(u'invalid gender entry: %s' % value) return if known[value] in (u'somevalue', u'novalue'): return WD.Statement(known[value], special=True) else: return WD.Statement(self.wd.QtoItemPage(known[value])) def db_name(self, name_obj, typ, limit=75): """Check if there is an item matching the name. A wrapper for helpers.match_name() to send it the relevant part of a nameObj. @param nameObj: {'@language': 'xx', '@value': 'xxx'} @type nameObj: dict @param typ: The name type (either 'lastName' or 'firstName') @type typ: str @param limit: Number of hits before skipping (defaults to 75, ignored if onLabs) @type limit: int @return: A matching item, if any @rtype: pywikibot.ItemPage, or None """ return helpers.match_name(name_obj['@value'], typ, self.wd, limit=limit) def location2Wikidata(self, uuid): """ Get location from kulturNav uuid. Given a kulturNav uuid or url this checks if that contains a GeoNames url and, if so, connects that to a Wikidata object using the GEONAMES_ID_P property (if any). NOTE that the WDQ results may be outdated return pywikibot.ItemPage|None """ # Check if uuid if not self.is_uuid(uuid): return None # Convert url to uuid if uuid.startswith(u'http://kulturnav.org'): uuid = uuid.split('/')[-1] # Check if already stored if uuid in self.locations.keys(): if self.locations[uuid] is None: return None else: qNo = u'Q%d' % self.locations[uuid] return self.wd.QtoItemPage(qNo) # retrieve various sources # @todo: this can be more streamlined by including wdq query for geonames # in that method. Possibly sharing the same "look-up and filter" # mechanism for both. # and then using self.locations[uuid] = self.extract... (which # returns qid or None) then (after both have been processed) # checking self.locations.get(uuid) before # making an ItemPage # # @todo: change self.locations and self.ADMIN_UNITS to include Q prefix (and thus have the methods return that) geo_sources = self.get_geo_sources(uuid) kulturarvsdata = self.extract_kulturarvsdata_location(geo_sources) if kulturarvsdata: self.locations[uuid] = kulturarvsdata qNo = u'Q%d' % self.locations[uuid] return self.wd.QtoItemPage(qNo) # retrieve hit through geonames-lookup geonames = KulturnavBot.extract_geonames(geo_sources) if geonames: # store as a resolved hit, in case wdq yields nothing self.locations[uuid] = None wdqQuery = u'STRING[%s:"%s"]' % (self.GEONAMES_ID_P, geonames) wdqResult = wdqsLookup.wdq_to_wdqs(wdqQuery) if wdqResult and len(wdqResult) == 1: self.locations[uuid] = wdqResult[0] qNo = u'Q%d' % self.locations[uuid] return self.wd.QtoItemPage(qNo) # else: # go to geonames and find wikidata from there # add to self.locations[uuid] # add GEONAMES_ID_P to the identified wikidata # no (clean) hits return None def get_geo_sources(self, uuid): """Extract any geosources from a kulturNav uuid. Given a kulturNav uuid return the corresponding properties of that target which are likely to contain geosources. @param uuid: uuid to check @type uuid: str @return: the matching properties @rtyp: list of dicts """ # debugging if not self.is_uuid(uuid): return [] query_url = 'http://kulturnav.org/api/%s' json_data = json.load(urllib2.urlopen(query_url % uuid)) sources = [] if json_data.get(u'properties'): same_as = json_data.get('properties').get('entity.sameAs') if same_as: sources += same_as source_uri = json_data.get('properties') \ .get('superconcept.sourceUri') if source_uri: sources += source_uri return sources @staticmethod def extract_geonames(sources): """Return any geonames ID given a list of get_geo_sources(). @param sources: output of get_geo_sources() @type sources: list of dicts @return: geonames id @rtype: str or None """ needle = 'http://sws.geonames.org/' for s in sources: if s.get('value') and s.get('value').startswith(needle): return s.get('value').split('/')[-1] return None def extract_kulturarvsdata_location(self, sources): """Return any qids matching kulturarvsdata geo authorities. @param sources: output of get_geo_sources() @type sources: list of dicts @return: the matching qid (without Q-prefix) @rtype: str or None @raises pywikibot.Error """ needle = u'http://kulturarvsdata.se/resurser/aukt/geo/' for s in sources: if s.get('value') and s.get('value').startswith(needle): s = s.get('value').split('/')[-1] wdq_query = None if s.startswith('municipality#'): code = s.split('#')[-1] wdq_query = u'STRING[%s:"%s"]' % (self.SWE_KOMMUNKOD_P, code) elif s.startswith('county#'): code = s.split('#')[-1] wdq_query = u'STRING[%s:"%s"]' % (self.SWE_COUNTYKOD_P, code) elif s.startswith('country#'): pass # handle via geonames instead elif s.startswith('parish#'): pass # no id's in wikidata else: raise pywikibot.Error(u'Unhandled KulturarvsdataLocation ' u'prefix: %s' % s) if wdq_query: # only here if a municipality or county was found wdq_result = wdqsLookup.wdq_to_wdqs(wdq_query) if wdq_result and len(wdq_result) == 1: self.ADMIN_UNITS.append(wdq_result[0]) return wdq_result[0] return None def getLocationProperty(self, item, strict=True): """ Return appropriate location property for an item. Given an ItemPage this returns the suitable property which should be used to indicate its location. P17 - land P131 - within administrative unit P276 - place param item: pywikibot.ItemPage|None param strict: bool whether place should be returned if no land or admin_unit hit return string|None """ if item is not None: q = int(item.title()[1:]) if q in self.COUNTRIES: return u'P17' elif q in self.ADMIN_UNITS: return u'P131' elif not strict: return u'P%s' % self.PLACE_P elif self.verbose: item.exists() pywikibot.output(u'Could not set location property for: ' u'%s (%s)' % (item.title(), item.labels.get('sv'))) return None def kulturnav2Wikidata(self, uuid): """Return Wikidata entity connected to a kulturNav uid or url. Relies on the KULTURNAV_ID_P property (if any) to get the connection. NOTE that the WDQ results may be outdated @param uuid: a kulturNav uuid or url @type uuid: str @return: the matching Wikidata item page @rtype: pywikibot.ItemPage or None """ # debugging if not self.is_uuid(uuid): return None # Convert url to uuid if uuid.startswith(u'http://kulturnav.org'): uuid = uuid.split('/')[-1] if uuid in self.itemIds.keys(): qNo = u'Q%d' % self.itemIds[uuid] return self.wd.QtoItemPage(qNo) else: return None def is_uuid(self, uuid): """Test if a string really is a uuid. @param uuid: uuid to test @type uuid: str @return: whether the test passed @rtype: bool """ if not helpers.is_str(uuid): pywikibot.output(u'Not an uuid in %s: %s' % (self.current_uuid, uuid)) return False uuid = uuid.split('/')[-1] # in case of url pattern = r'[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}' \ r'\-[0-9a-f]{4}\-[0-9a-f]{12}' m = re.search(pattern, uuid) if not m or m.group(0) != uuid: pywikibot.output(u'Not an uuid in %s: %s' % (self.current_uuid, uuid)) return False return True @staticmethod def shuffle_names(name_obj): """Detect a "Last, First" string and return as "First Last". A wrapper for helpers.reorder_names() to send it the relevant part of a name_obj. @param name_obj: {'@language': 'xx', '@value': 'xxx'} @type name_obj: dict @return: the reordered name_obj or None if reorder_names failed @rtype: dict or None """ name = helpers.reorder_names(name_obj['@value']) if name is None: return None name_obj = name_obj.copy() name_obj['@value'] = name return name_obj def make_ref(self, date): """Make a correctly formatted ref object for claims. Contains 4 parts: * P248: Stated in <the kulturnav dataset> * P577: Publication date <from the document> * P854: Reference url <using the current uuid> * P813: Retrieval date <current date> P854 Should be in source_test (after retroactively fixing older references) but by being in source_notest we ensure that duplicate uuids don't source the statement twice. @param date: The "last modified" time of the document @type date: pywikibot.WbTime @return: the formated reference @rtype WD.Reference """ reference_url = 'http://kulturnav.org/%s' % self.current_uuid ref = WD.Reference( source_test=self.wd.make_simple_claim( 'P248', self.wd.QtoItemPage(self.DATASET_Q)), source_notest=[ self.wd.make_simple_claim('P577', date), self.wd.make_simple_claim('P854', reference_url), self.wd.make_simple_claim('P813', helpers.today_as_WbTime()) ]) return ref def add_label_or_alias(self, name_obj, item, case_sensitive=False): """Add a name as either a label (if none already) or an alias. Essentially a filter for the more generic method in WikidatStuff. @param name_obj: {'@language': 'xx', '@value': 'xxx'} or a list of such @type name_obj: dict or list of dict @param item: the item to which the label/alias should be added @type item: pywikibot.ItemPage @param caseSensitive: whether the comparison is case sensitive @type caseSensitive: bool """ # for a list of entries if isinstance(name_obj, list): for n in name_obj: self.add_label_or_alias(n, item, case_sensitive=case_sensitive) # reload item so that next call is aware of any changes item = self.wd.QtoItemPage(item.title()) item.exists() return # for a single entry self.wd.addLabelOrAlias(name_obj['@language'], name_obj['@value'], item, caseSensitive=case_sensitive) @staticmethod def get_kulturnav_generator(uuids, delay=0): """Generate KulturNav items from a list of uuids. @param uuids: uuids to request items for @type uuids: list of str @param delay: delay in seconds between each kulturnav request @type delay: int @yield: dict """ for uuid in uuids: time.sleep(delay) try: json_data = KulturnavBot.get_single_entry(uuid) except pywikibot.Error as e: pywikibot.output(e) else: yield json_data @classmethod def get_search_results(cls, max_hits=250, require_wikidata=True): """Make a KulturNav search for all items of a given type in a dataset. @param max_hits: the maximum number of results to request at once @type max_hits: int @param require_wikidata: whether to filter results on having a wikidata url in sameAs @type require_wikidata: bool @return: the resulting uuids @rtype: list of str """ search_url = 'http://kulturnav.org/api/search/' + \ 'entityType:%s,' % cls.ENTITY_TYPE + \ 'entity.dataset_r:%s' % cls.DATASET_ID q = None # the map_tag query # only filter on MAP_TAG if filtering on wikidata if require_wikidata: search_url += ',%s' % cls.MAP_TAG + ':%s/%d/%d' q = '*%2F%2Fwww.wikidata.org%2Fentity%2FQ*' else: search_url += '/%d/%d' # start search results = [] offset = 0 overview_page = KulturnavBot.get_single_search_results( search_url, q, offset, max_hits) while overview_page: for item in overview_page: uuid = item[u'uuid'] if not require_wikidata or \ KulturnavBot.has_wikidata_in_sameas(item, cls.MAP_TAG): results.append(uuid) # continue offset += max_hits overview_page = KulturnavBot.get_single_search_results( search_url, q, offset, max_hits) # some feedback pywikibot.output(u'Found %d matching entries in Kulturnav' % len(results)) return results @staticmethod def has_wikidata_in_sameas(item, map_tag): """Check if a wikidata url is present in the sameAs property. @param item: the search item to check @type item: dict @param map_tag: the tag to use (concepts don't use sameAs) @type map_tag: str @rtype: bool """ # The patterns used if we filter on wikidata patterns = (u'http://www.wikidata.org/entity/', u'https://www.wikidata.org/entity/') same_as = item[u'properties'][map_tag[:map_tag.rfind('_')]] for s in same_as: if s[u'value'].startswith(patterns): return True return False @staticmethod def get_single_search_results(search_url, q, offset, max_hits): """Retrieve the results from a single API search. @param search_url: basic url from whih to build search @type search_url: str @param q: the map_tag query, if any @type q: str or None @param offset: the offset in search results @type offset: int @param max_hits: the maximum number of results to request at once @type max_hits: int @return: the search result object @rtype: dict """ actual_url = '' if q is None: actual_url = search_url % (offset, max_hits) else: actual_url = search_url % (q, offset, max_hits) search_page = urllib2.urlopen(actual_url) return json.loads(search_page.read()) @staticmethod def get_single_entry(uuid): """Retrieve the data on a single kulturnav entry. Raises an pywikibot.Error if: * @graph is not a key in the json response * a non-json response is received @param uuid: the uuid for the target item @type uuid: str @return: the entry object @rtype: dict @raise: pywikibot.Error """ query_url = 'http://kulturnav.org/%s?format=application/ld%%2Bjson' item_url = query_url % uuid try: record_page = urllib2.urlopen(item_url) json_data = json.loads(record_page.read()) except ValueError as e: raise pywikibot.Error('Error loading KulturNav item at ' '%s with error %s' % (item_url, e)) if json_data.get(u'@graph'): return json_data else: raise pywikibot.Error('No @graph in KulturNav reply at ' '%s\n data: %s' % (item_url, json_data)) @classmethod def main(cls, *args): """Start the bot from the command line.""" options = cls.handle_args(args) search_results = cls.get_search_results( max_hits=options['max_hits'], require_wikidata=options['require_wikidata']) kulturnav_generator = cls.get_kulturnav_generator( search_results, delay=options['delay']) kulturnav_bot = cls(kulturnav_generator, options['cache_max_age']) kulturnav_bot.cutoff = options['cutoff'] kulturnav_bot.require_wikidata = options['require_wikidata'] kulturnav_bot.run() @classmethod def run_from_list(cls, uuids, *args): """Start the bot with a list of uuids.""" options = cls.handle_args(args) kulturnav_generator = cls.get_kulturnav_generator( uuids, delay=options['delay']) kulturnav_bot = cls(kulturnav_generator, options['cache_max_age']) kulturnav_bot.cutoff = options['cutoff'] kulturnav_bot.require_wikidata = False kulturnav_bot.run() @staticmethod def handle_args(args): """Parse and load all of the basic arguments. Also passes any needed arguments on to pywikibot and sets any defaults. @param args: arguments to be handled @type args: list of strings @return: list of options @rtype: dict """ options = { 'cutoff': None, 'max_hits': 250, 'delay': 0, 'require_wikidata': True, 'cache_max_age': 0, } for arg in pywikibot.handle_args(args): option, sep, value = arg.partition(':') if option == '-cutoff': options['cutoff'] = int(value) elif option == '-max_hits': options['max_hits'] = int(value) elif option == '-delay': options['delay'] = int(value) elif option == '-any_item': options['require_wikidata'] = False elif option == '-wdq_cache': options['cache_max_age'] = int(value) return options @staticmethod def foobar(item): """Badly named escape mechanism for list results.""" if isinstance(item, list): pywikibot.output(FOO_BAR) return True return False
class PaintingsImageBot: """Bot to enrich, and create, for items about paintings on Wikidata.""" def __init__(self, dict_generator, people_items): """Initialise the bot.""" self.people_items = people_items self.generator = dict_generator self.repo = pywikibot.Site().data_repository() self.wd = WD(self.repo, edit_summary=EDIT_SUMMARY) # Set log file out_dir = path.join(path.split(__file__)[0]) log_filename = path.join(out_dir, u'PaintingsImageBot.log') self.log = codecs.open(log_filename, 'a', 'utf-8') def run(self): """Start the robot.""" self.creators = {} for painting_data in self.generator: # isolate ids lido_data, qid, commons_file = painting_data painting_item = self.wd.QtoItemPage(qid) self.process_painting(painting_item, lido_data, commons_file) def process_painting(self, item, lido_data, commons_file): """Process a single painting.""" item.exists() # load the item obj_id_ref = self.make_obj_id_ref(lido_data.get('obj_id')) # lido_ref = self.make_lido_ref(lido_data) # make a reference object self.check_and_add_labels(item, lido_data) self.add_image_claim(item, commons_file, obj_id_ref) self.add_depicted_claim(item, lido_data, obj_id_ref) self.add_date_claim(item, lido_data, obj_id_ref) self.add_dimension_claims(item, lido_data, obj_id_ref) def add_dimension_claims(self, item, lido_data, ref): """ Add height/P2048 and width/P2049 claims. Only add non-framed measurements with just height and width. """ height_p = u'P2048' width_p = u'P2049' # diameter_p = u'P2386' # thickness_p = u'P2610' dimensions = lido_data.get('measurements').get('_') # non-framed if not dimensions or not dimensions.get('unit'): return None elif not dimensions.get('width') or not dimensions.get('height') \ or dimensions.get('depth'): # skip complicated cases for now return None elif not helpers.get_unit_q(dimensions.get('unit')): pywikibot.output(u'"%s" is an unmapped unit' % dimensions.get('unit')) return None # prepare all parts before adding claims unit = helpers.get_unit_q(dimensions.get('unit')) # unit = self.wd.QtoItemPage(unit) unit = entity_url_hack(unit) height = pywikibot.WbQuantity( dimensions.get('height'), # unit=unit, entity=unit, site=self.wd.repo) width = pywikibot.WbQuantity( dimensions.get('width'), # unit=unit, entity=unit, site=self.wd.repo) # make claims self.wd.addNewClaim(height_p, WD.Statement(height), item, ref) self.wd.addNewClaim(width_p, WD.Statement(width), item, ref) def add_date_claim(self, item, lido_data, ref): """ Add an inception/P571 claim. Only adds the claim if it's an exact year. """ prop = u'P571' creation_date = lido_data.get('creation_date') wb_date = None if not creation_date: return None # exact date if creation_date.get('earliest') and \ creation_date.get('earliest') == creation_date.get('latest'): wb_date = helpers.iso_to_WbTime(creation_date.get('earliest')) # make claim if wb_date: self.wd.addNewClaim(prop, WD.Statement(wb_date), item, ref) def add_depicted_claim(self, item, lido_data, ref): """Add a depicted/P180.""" prop = u'P180' if not lido_data.get('subjects'): return None for subject in lido_data.get('subjects'): nsid = subject.get(u'other_id') if nsid in self.people_items: person_item = self.wd.QtoItemPage(self.people_items[nsid]) self.wd.addNewClaim(prop, WD.Statement(person_item), item, ref) def add_image_claim(self, item, commons_file, ref): """ Add a image/P18 claim. Only adds it if there is None already. If one exists output to log. """ prop = u'P18' if not commons_file: return file_page = pywikibot.FilePage(pywikibot.Site('commons', 'commons'), commons_file) # check if another image is already used if prop in item.claims and \ not self.wd.has_claim(prop, file_page, item): self.log.write( u"%s already contains image claim: %s -> %s\n" % (item.title(), item.claims.get(prop)[0].getTarget().title(), file_page.title())) else: self.wd.addNewClaim(prop, WD.Statement(file_page), item, ref) def check_and_add_labels(self, item, lido_data): """Process the title field add to the item if needed.""" if not lido_data.get('title'): return for lang, value in lido_data.get('title').iteritems(): if lang == '_': continue try: self.wd.addLabelOrAlias(lang, value, item, caseSensitive=False) except pywikibot.data.api.APIError as e: self.log.write(u"%s: had an error: %s\n" % (item.title(), e)) def make_obj_id_ref(self, obj_id): """Make a reference object pointing to the objects collection page.""" uri = u'http://collection.nationalmuseum.se/eMuseumPlus?' \ u'service=ExternalInterface&module=collection&' \ u'objectId=%s&viewType=detailView' % obj_id return self.make_url_reference(uri) def make_url_reference(self, uri): """ Make a Reference object with a retrieval url and today's date. @param uri: retrieval uri/url @type uri: str @rtype: WD.Reference """ date = helpers.today_as_WbTime() ref = WD.Reference(source_test=self.wd.make_simple_claim(u'P854', uri), source_notest=self.wd.make_simple_claim( u'P813', date)) return ref # Not implemented due to uncertainty on referencing individual xml files def make_lido_ref(self, lido_data): """ Make a Reference object for the dataset. Contains 4 parts: * P248: Stated in <the Nationalmuseum dataset> * P577: Publication date <from creation date of the document> * P854: Reference url <using the input url> * P813: Retrieval date <current date> """ exit() # P248: Nationalmuseum dataset xml_file = lido_data.get('source_file') date = helpers.today_as_WbTime() pub_date = helpers.iso_to_WbTime(u'2016-09-30') zip_url = u'https://github.com/NationalmuseumSWE/WikidataCollection/' \ u'blob/master/valid_items_transform_1677.tgz' ref = WD.Reference(source_test=[ self.wd.make_simple_claim(u'P854', zip_url), self.wd.make_simple_claim(u'P577', pub_date), self.wd.make_simple_claim(u'P?', xml_file), ], source_notest=self.wd.make_simple_claim( u'P813', date)) return ref