def _same_tree(cls, prop, data1, data2): sparql = SparqlQuery() # fixme: dependencies pattern = ('ASK { VALUES ?x1 { wd:%s } . VALUES ?x2 { wd:%s } . ' '?x1 wdt:%s* ?x2 }') item1 = ' wd:'.join(map(attrgetter('target.id'), data1)) item2 = ' wd:'.join(map(attrgetter('target.id'), data2)) tries = 3 for ask in (pattern % (item1, item2, prop), pattern % (item2, item1, prop)): res = False while True: try: res = sparql.ask(ask) except requests.exceptions.ConnectionError: tries -= 1 if tries == 0: raise time.sleep(1) continue else: break if res: return True return False
def _same_tree(cls, prop, data1, data2): sparql = SparqlQuery() # fixme: dependencies pattern = ('ASK { VALUES ?x1 { wd:%s } . VALUES ?x2 { wd:%s } . ' '?x1 wdt:%s* ?x2 }') item1 = ' wd:'.join(map(attrgetter('target.id'), data1)) item2 = ' wd:'.join(map(attrgetter('target.id'), data2)) tries = 3 for ask in (pattern % (item1, item2, prop), pattern % (item2, item1, prop)): res = False while True: try: res = sparql.ask(ask) except requests.exceptions.ConnectionError: tries -= 1 if tries == 0: raise time.sleep(1) continue else: break if res: return True return False
class ExternalIdSlicingBot(WikidataEntityBot): blacklist = {'P2013'} use_from_page = False def __init__(self, **options): self.available_options.update({ 'step': 10, 'offset': 0, }) super().__init__(**options) self.cache = {} self.failed = {} self.sparql = SparqlQuery(repo=self.repo) self.store = QueryStore() @property def generator(self): step = self.opt['step'] opts = { # fixme: don't use this word 'blacklist': ' wd:'.join(self.blacklist), 'limit': step, } offset = self.opt['offset'] while True: pywikibot.output('\nLoading items (offset %i)...' % offset) opts['offset'] = offset ask = self.store.build_query('ask_externalid_props', **opts) if not self.sparql.ask(ask): break query = self.store.build_query('external-ids', **opts) gen = PreloadingEntityGenerator( WikidataSPARQLPageGenerator(query, site=self.repo)) yield from gen offset += step def treat_page_and_item(self, page, item): for prop, claims in item.claims.items(): if prop in self.blacklist: continue if claims[0].type != 'external-id': continue for cl in claims: if not cl.target or not cl.target.startswith('http'): continue formatter, regex = self.get_formatter_and_regex(prop) if not formatter: pywikibot.output("%s doesn't have a formatter" % prop) break value = self.find_value(cl.target, formatter) if not value: pywikibot.output( 'Value not found in "%s" for property %s' % (cl.target, prop)) self.failed.setdefault(prop, set()).add(item) continue if regex: try: match = re.match('(%s)' % regex, value) except re.error: pywikibot.output('Couldn\'t apply regex "%s"' % regex) break if not match: pywikibot.output('Value "%s" not matched by regex ' '"%s"' % (value, regex)) self.failed.setdefault(prop, set()).add(item) continue value = match.group() summary = 'harvested the identifier based on [[Property:P1630]]' if regex: summary += ' and [[Property:P1793]]' cl.changeTarget(value, summary=summary) def get_formatter_and_regex(self, prop): if prop not in self.cache: formatter = regex = None ppage = pywikibot.PropertyPage(self.repo, prop) if 'P1630' in ppage.claims: if len(ppage.claims['P1630']) > 1: preferred = [ cl for cl in ppage.claims['P1630'] if cl.rank == 'preferred' ] if len(preferred) == 1: formatter = preferred[0].target else: formatter = ppage.claims['P1630'][0].target if 'P1793' in ppage.claims: if len(ppage.claims['P1793']) > 1: preferred = [ cl for cl in ppage.claims['P1793'] if cl.rank == 'preferred' ] if len(preferred) == 1: regex = preferred[0].target else: regex = ppage.claims['P1793'][0].target self.cache[prop] = (formatter, regex) return self.cache[prop] def strip_init_stuff(self, string): if string.startswith(('http://', 'https://')): string = string.partition('//')[2] if string.startswith('www.'): string = string[4:] return string def find_value(self, url, formatter): url = self.strip_init_stuff(url) formatter = self.strip_init_stuff(formatter) value = pywikibot.page.url2unicode(url) split = formatter.split('$1') if not value.startswith(split[0]): return None if not split[1]: return value[len(split[0]):].rstrip('/') value = value[:-len(split[-1])] try: index = value.index(split[1], len(split[0])) except ValueError: return None else: return value[len(split[0]):index].rstrip('/') def exit(self): if self.failed: text = '' for prop in sorted(self.failed): text += '* [[Property:%s]]:\n' % prop for item in sorted(self.failed[prop]): text += '** [[%s]]\n' % item.title() page = pywikibot.Page( self.repo, 'User:%s/Wrong external ids' % self.repo.username()) page.put(text, summary='update') super().exit()
class DuosManagingBot(WikidataEntityBot): conj = { 'af': ' en ', 'az': ' və ', 'bg': ' и ', 'br': ' ha ', 'ca': ' i ', 'cs': ' a ', 'cy': ' a ', 'da': ' og ', 'de': ' und ', 'el': ' και ', 'en': ' and ', 'en-gb': ' and ', 'eo': ' kaj ', 'es': ' y ', 'et': ' ja ', 'eu': ' eta ', 'fi': ' ja ', 'fr': ' et ', 'fy': ' en ', 'gl': ' e ', 'hr': ' i ', 'hu': ' és ', 'id': ' dan ', 'it': ' e ', 'ka': ' და ', 'la': ' et ', 'lt': ' ir ', 'lv': ' un ', 'ms': ' dan ', 'nb': ' og ', 'nl': ' en ', 'nn': ' og ', 'oc': ' e ', 'pl': ' i ', 'pt': ' e ', 'ro': ' și ', 'ru': ' и ', 'sk': ' a ', 'sl': ' in ', 'sr': ' и ', 'sv': ' och ', 'tr': ' ve ', 'uk': ' і ', 'vi': ' và ', 'war': ' ngan ', } distribute_properties = [ 'P21', 'P22', 'P25', 'P27', 'P40', 'P53', 'P106', 'P1412', ] class_to_relation = [ ('Q106925878', 'father-son'), ('Q14756018', 'twin'), ('Q14073567', 'sibling'), ('Q3046146', 'spouse'), # TODO: ('Q1141470', 'comedians'), not a "relation by blood" ] relation_map = { 'sibling': 'P3373', 'spouse': 'P26', 'twin': 'P3373', # TODO: 'partner': 'P451', #'father-son': '', we don't know who is who #'comedians': 'P1327', } use_from_page = False def __init__(self, generator, **kwargs): self.available_options.update({ 'always': True, 'class': 'Q10648343', 'min_labels': 1, }) super().__init__(**kwargs) self.store = QueryStore() self.sparql = SparqlQuery(repo=self.repo) self._generator = generator or self.custom_generator() def skip_page(self, item): if super().skip_page(item): return True if 'P31' not in item.claims: pywikibot.output('%s is missing P31 property' % item) return True if 'P527' in item.claims: pywikibot.output('%s already has P527 property' % item) return True return False def custom_generator(self): kwargs = {'class': self.opt['class']} query = self.store.build_query('duos', **kwargs) return pagegenerators.WikidataSPARQLPageGenerator(query, site=self.repo) @property def generator(self): return pagegenerators.PreloadingEntityGenerator(self._generator) def get_relation(self, item): ask_pattern = 'ASK { wd:%s wdt:P31/wdt:P279* wd:%%s }' % item.id for key, rel in self.class_to_relation: if self.sparql.ask(ask_pattern % key): return rel return None def get_labels(self, item, relation): labels = [{}, {}] for lang in item.labels.keys() & self.conj.keys(): for conj in (self.conj[lang], ' & '): label = item.labels[lang].partition(' (')[0] if ', ' in label: continue split = label.split(conj) if len(split) != 2: continue split0 = split[0].split() split1 = split[1].split() if split1[0].islower(): continue # TODO: if len(split1) > 1 and split1[0][-1] == '.': if len(split1) > len(split0): if len(split1) > 2 and split1[-2].islower(): split1[-2:] = [' '.join(split1[-2:])] if len(split1) - len(split0) == 1: # if items are in a relation, then # they probably share their surname if relation: split[0] += ' %s' % split1[-1] split0.append(split1[-1]) if len(split0) > 1 or len(split1) == 1: labels[0][lang] = split[0] labels[1][lang] = split[1] break return labels def treat_page_and_item(self, page, item): relation = self.get_relation(item) labels = self.get_labels(item, relation) count = max(map(len, labels)) if count == 0: pywikibot.output('No labels, skipping...') return if count < self.opt['min_labels']: pywikibot.output('Too few labels (%i), skipping...' % count) return to_add = [] to_remove = [] for prop in self.distribute_properties: for claim in item.claims.get(prop, []): if claim.getTarget(): to_remove.append(claim) json = claim.toJSON() json.pop('id') to_add.append(json) items = [self.create_item(item, data, relation, to_add) for data in labels] if self.relation_map.get(relation): for it, target in zip(items, reversed(items)): claim = pywikibot.Claim(self.repo, self.relation_map[relation]) claim.setTarget(target) self.user_add_claim(it, claim) for it in items: claim = pywikibot.Claim(self.repo, 'P527') claim.setTarget(it) self.user_add_claim(item, claim) for claim in to_remove: pywikibot.output('Removing %s --> %s' % ( claim.id, claim.getTarget())) json = claim.toJSON() json['remove'] = '' summary = 'moved [[Property:{}]] to {} & {}'.format( claim.id, items[0].title(as_link=True, insite=self.repo), items[1].title(as_link=True, insite=self.repo) ) self.user_edit_entity(item, {'claims':[json]}, summary=summary) def create_item(self, item, labels, relation, to_add): pywikibot.output('Creating item (relation "%s")...' % relation) new_item = pywikibot.ItemPage(self.repo) self.user_edit_entity( new_item, {'labels': labels}, asynchronous=False, summary='based on data in %s' % item.title( as_link=True, insite=self.repo)) claim = pywikibot.Claim(self.repo, 'P31') claim.setTarget(pywikibot.ItemPage(self.repo, 'Q5')) self.user_add_claim(new_item, claim) claim = pywikibot.Claim(self.repo, 'P361') claim.setTarget(item) self.user_add_claim(new_item, claim) for json in to_add: temp_claim = pywikibot.Claim.fromJSON(self.repo, json) pywikibot.output('Adding %s --> %s' % ( temp_claim.id, temp_claim.getTarget())) self.user_edit_entity( new_item, {'claims':[json]}, summary='moving [[Property:%s]] from %s' % ( temp_claim.id, item.title(as_link=True, insite=self.repo))) return new_item
class DuosManagingBot(WikidataEntityBot): conj = { 'af': ' en ', 'az': ' və ', 'bg': ' и ', 'br': ' ha ', 'ca': ' i ', 'cs': ' a ', 'cy': ' a ', 'da': ' og ', 'de': ' und ', 'el': ' και ', 'en': ' and ', 'en-gb': ' and ', 'eo': ' kaj ', 'es': ' y ', 'et': ' ja ', 'eu': ' eta ', 'fi': ' ja ', 'fr': ' et ', 'fy': ' en ', 'gl': ' e ', 'hr': ' i ', 'hu': ' és ', 'id': ' dan ', 'it': ' e ', 'ka': ' და ', 'la': ' et ', 'lt': ' ir ', 'lv': ' un ', 'ms': ' dan ', 'nb': ' og ', 'nl': ' en ', 'nn': ' og ', 'oc': ' e ', 'pl': ' i ', 'pt': ' e ', 'ro': ' și ', 'ru': ' и ', 'sk': ' a ', 'sl': ' in ', 'sr': ' и ', 'sv': ' och ', 'tr': ' ve ', 'uk': ' і ', 'vi': ' và ', 'war': ' ngan ', } distribute_properties = { 'P21', 'P22', 'P25', 'P27', 'P40', 'P53', 'P106', 'P1412', } class_to_relation = [ ('Q14756018', 'twin'), ('Q14073567', 'sibling'), ('Q3046146', 'spouse'), ] relation_map = { #'partner': 'P451', todo 'sibling': 'P3373', 'spouse': 'P26', 'twin': 'P3373', } use_from_page = False def __init__(self, generator, **kwargs): self.availableOptions.update({ 'always': True, 'class': 'Q15618652', 'min_labels': 1, }) super(DuosManagingBot, self).__init__(**kwargs) self.store = QueryStore() self.sparql = SparqlQuery(repo=self.repo) self._generator = generator or self.custom_generator() def skip_page(self, item): if super(DuosManagingBot, self).skip_page(item): return True if 'P31' not in item.claims: pywikibot.output('%s is missing P31 property' % item) return True if 'P527' in item.claims: pywikibot.output('%s already has P527 property' % item) return True return False def custom_generator(self): kwargs = {'class': self.getOption('class')} query = self.store.build_query('duos', **kwargs) return pagegenerators.WikidataSPARQLPageGenerator(query, site=self.repo) @property def generator(self): return pagegenerators.PreloadingEntityGenerator(self._generator) def get_relation(self, item): ask_pattern = 'ASK { wd:%s wdt:P31/wdt:P279* wd:%%s }' % item.id for key, rel in self.class_to_relation: if self.sparql.ask(ask_pattern % key): return rel return None def get_labels(self, item, relation): labels = [{}, {}] for lang in set(item.labels.keys()) & set(self.conj.keys()): for conj in (self.conj[lang], ' & '): label = item.labels[lang].partition(' (')[0] if ', ' in label: continue split = label.split(conj) if len(split) != 2: continue split0 = split[0].split() split1 = split[1].split() if split1[0].islower(): continue if len(split1) > len(split0): if len(split1) > 2 and split1[-2].islower(): split1[-2:] = [' '.join(split1[-2:])] if len(split1) - len(split0) == 1: # if items are in a relation, then they probably share # their surname if relation: split[0] += ' %s' % split1[-1] split0.append(split1[-1]) if len(split0) > 1 or len(split1) == 1: for i in [0, 1]: labels[i][lang] = split[i] break return labels def treat_page_and_item(self, page, item): relation = self.get_relation(item) labels = self.get_labels(item, relation) count = max(map(len, labels)) if count == 0: pywikibot.output('No labels, skipping...') return if count < self.getOption('min_labels'): pywikibot.output('Too few labels (%i), skipping...' % count) return to_add = [] to_remove = [] for prop in self.distribute_properties: for claim in item.claims.get(prop, []): if claim.getTarget(): to_remove.append(claim) json = claim.toJSON() json.pop('id') to_add.append(json) items = [self.create_item(item, data, relation, to_add) for data in labels] if self.relation_map.get(relation): for it, target in zip(items, reversed(items)): claim = pywikibot.Claim(self.repo, self.relation_map[relation]) claim.setTarget(target) self.user_add_claim(it, claim) for it in items: claim = pywikibot.Claim(self.repo, 'P527') claim.setTarget(it) self.user_add_claim(item, claim) for claim in to_remove: pywikibot.output('Removing %s --> %s' % ( claim.id, claim.getTarget())) json = claim.toJSON() json['remove'] = '' self.user_edit_entity( item, {'claims':[json]}, summary='moved [[Property:%s]] to %s' % ( claim.id, ' & '.join(map(methodcaller( 'title', as_link=True, insite=self.repo), items)))) def create_item(self, item, labels, relation, to_add): pywikibot.output('Creating item (relation "%s")...' % relation) new_item = pywikibot.ItemPage(self.repo) data = {'labels': labels} self.user_edit_entity( new_item, data, summary='based on data in %s' % item.title( as_link=True, insite=self.repo), asynchronous=False) claim = pywikibot.Claim(self.repo, 'P31') claim.setTarget(pywikibot.ItemPage(self.repo, 'Q5')) self.user_add_claim(new_item, claim) ## if relation == 'twin': ## claim = pywikibot.Claim(self.repo, 'P31') ## claim.setTarget(pywikibot.ItemPage(self.repo, 'Q159979')) ## self.user_add_claim(new_item, claim) claim = pywikibot.Claim(self.repo, 'P361') claim.setTarget(item) self.user_add_claim(new_item, claim) for json in to_add: temp_claim = pywikibot.Claim.fromJSON(self.repo, json) pywikibot.output('Adding %s --> %s' % ( temp_claim.id, temp_claim.getTarget())) self.user_edit_entity( new_item, {'claims':[json]}, summary='moving [[Property:%s]] from %s' % ( temp_claim.id, item.title(as_link=True, insite=self.repo))) return new_item