def claims(self, values): """Add protoclaims. @param values: the values extracted using the rules @type values: dict @return: the protoclaims @rtype: dict PID-WD.Statement pairs """ # handle altNames together with names values[u'entity.name'] = KulturnavBotSMM.prep_names(values) # bundle type and otherType values[u'navalVessel.type'] = KulturnavBotSMM.prep_types(values) protoclaims = { # operator = Swedish Navy u'P137': WD.Statement(self.wd.QtoItemPage(self.SWENAVY_Q)) } # P31 - instance of # ship class unless a submarine class_Q = self.SHIPCLASS_Q if values[u'navalVessel.type'] and \ any(x.endswith(self.SUBMARINETYPE_K) for x in values[u'navalVessel.type']): class_Q = self.SUBMARINECLASS_Q protoclaims[u'P31'] = WD.Statement(self.wd.QtoItemPage(class_Q)) # P279 - subgroup self.set_subgroup(values, protoclaims) # P287 - Designer (Constructor) self.set_constructor(values, protoclaims) return protoclaims
def set_type_and_class(self, values, protoclaims): """Identify type (P31) and class (P289) and add to claims. Adds the claim to the protoclaims dict. @param values: the values extracted using the rules @type values: dict @param protoclaims: the dict of claims to add @type protoclaims: dict """ if values.get(u'navalVessel.type'): ship_class = [] ship_type = [] for val in values[u'navalVessel.type']: item = self.kulturnav2Wikidata(val) if item: q = int(item.title()[1:]) if q in self.class_list: ship_class.append(WD.Statement(item)) elif q in self.type_list: ship_type.append(WD.Statement(item)) else: pywikibot.output(u'Q%d not matched as either ship' u'type or ship class' % q) if ship_class: protoclaims[u'P289'] = ship_class if ship_type: protoclaims[u'P31'] = ship_type
def set_ship_events(self, values, protoclaims): """Identify any events (P793) for a ship then add to claims. Adds the claim(s) to the protoclaims dict. Events are only added IFF they have an associated date. @todo: commissioned: Q14475832 @param values: the values extracted using the rules @type values: dict @param protoclaims: the dict of claims to add @type protoclaims: dict """ events = [] # built: Q474200 event = WD.Statement(self.wd.QtoItemPage('Q474200')) if self.set_date_qualifier(values, 'built', event, prop=helpers.END_P): self.set_location_qualifier(values, 'built', event) # u'built.shipyard' events.append(event) # launched: Q596643 event = WD.Statement(self.wd.QtoItemPage('Q596643')) if self.set_date_qualifier(values, 'launched', event): # u'launched.shipyard' events.append(event) # decommissioned: Q7497952 event = WD.Statement(self.wd.QtoItemPage('Q7497952')) if self.set_date_qualifier(values, 'decommissioned', event): events.append(event) # set all events if events: protoclaims[u'P793'] = events
def set_creator(self, target_item, reference, creator_q=None, related_info=None): """Set a creator/P170 claim for a creator or creator combo. Allows for simple claims as well as more complex "in the manner of" etc. @param target_item: item to which claim is added @type target_item: pywikibot.ItemPage @param reference: the reference for the statment @type reference: WD.Reference @param related_info: related info as a dict with P/itis pairs @type related_info: dict @param creator_q: the Q-id of the creator @type creator_q: str """ creator_q = creator_q or ANON_Q creator_statement = WD.Statement(self.wd.QtoItemPage(creator_q)) # set any related qualifiers if related_info: creator_statement.addQualifier( WD.Qualifier(P=related_info['P'], itis=related_info['itis'])) # set claim self.wd.addNewClaim(u'P170', creator_statement, target_item, reference)
def add_start_end_qualifiers(statement, startVal, endVal): """ Add start/end qualifiers to a statement if non-None, or return None. @param statement: The statement to decorate @type statement: WD.Statement @param startVal: An ISO date string for the starting point @type startVal: basestring or None @param endVal: An ISO date string for the end point @type endVal: basestring or None @return: A statement decorated with start/end qualifiers @rtype: WD.Statement, or None """ if not isinstance(statement, WD.Statement): raise pywikibot.Error('Non-statement recieved: %s' % statement) if statement.isNone(): return None # add qualifiers quals = [] if startVal: quals.append( WD.Qualifier( P=START_P, itis=iso_to_WbTime(startVal))) if endVal: quals.append( WD.Qualifier( P=END_P, itis=iso_to_WbTime(endVal))) for q in quals: statement.addQualifier(q) return statement
def test_statement_equality_qualifier_order(self): s_1 = WD.Statement('foo') s_2 = WD.Statement('foo') s_3 = WD.Statement('foo') s_1.addQualifier(self.q_1).addQualifier(self.q_2) s_2.addQualifier(self.q_2).addQualifier(self.q_1) s_3.addQualifier(self.q_1) self.assertEquals(s_1, s_2) self.assertNotEquals(s_1, s_3)
def setUp(self): wikidata = Site('test', 'wikidata') self.q_1 = WD.Qualifier('P123', 'foo') self.q_2 = WD.Qualifier('P123', 'bar') claim = Claim(wikidata, 'P55') claim.setTarget('foo') self.ref = WD.Reference(source_test=[ claim, ])
def test_reference_get_all_sources(self): r_test = WD.Reference(source_test=self.ref_1) self.assertEqual(r_test.get_all_sources(), [self.ref_1]) r_notest = WD.Reference(source_notest=self.ref_1) self.assertEqual(r_notest.get_all_sources(), [self.ref_1]) r_both = WD.Reference(self.ref_1, self.ref_2) self.assertEqual(r_both.get_all_sources(), [self.ref_1, self.ref_2])
def test_qualifier_hash(self): q_1 = WD.Qualifier('P123', 'foo') q_2 = WD.Qualifier('P124', 'foo') q_3 = WD.Qualifier('P123', 'bar') q_same = WD.Qualifier('P123', 'foo') self.assertEqual(q_1, q_same) self.assertNotEqual(q_1, q_2) self.assertNotEqual(q_1, q_3) self.assertNotEqual(q_2, q_3)
def test_statement_inequality(self): s = WD.Statement('novalue') s_special = WD.Statement('novalue', special=True) s_force = WD.Statement('novalue') s_force.force = True self.assertNotEquals(s, s_special) self.assertNotEquals(s, s_force) self.assertNotEquals(s_force, s_special)
def test_statement_init_special(self): self.assertTrue(WD.Statement('somevalue', special=True).special) self.assertTrue(WD.Statement('novalue', special=True).special) with self.assertRaises(pwbError) as cm: WD.Statement('foo', special=True) self.assertEqual( str(cm.exception), 'You tried to create a special statement with a non-allowed ' 'snakvalue: foo')
def test_statement_equality(self): s = WD.Statement('foo') s_same = WD.Statement('foo') s_different = WD.Statement('bar') self.assertEquals(s, s) self.assertEquals(s, s_same) self.assertNotEquals(s, s_different) # Comparison with other classes always gives false, weird but expected self.assertFalse(s == 'foo') self.assertFalse(s != 'foo')
def test_reference_init_single_claim_gives_list(self): r_test = WD.Reference(source_test=self.ref_1) self.assertEqual(r_test.source_test, [self.ref_1]) self.assertEqual(r_test.source_notest, []) r_notest = WD.Reference(source_notest=self.ref_1) self.assertEqual(r_notest.source_test, []) self.assertEqual(r_notest.source_notest, [self.ref_1]) r_both = WD.Reference(self.ref_1, self.ref_2) self.assertEqual(r_both.source_test, [self.ref_1]) self.assertEqual(r_both.source_notest, [self.ref_2])
def test_reference_init_non_claim_error(self): with self.assertRaises(pwbError) as cm: WD.Reference(source_test='foo') self.assertEqual( str(cm.exception), 'You tried to create a reference with a non-Claim source') with self.assertRaises(pwbError) as cm: WD.Reference(source_notest='foo') self.assertEqual( str(cm.exception), 'You tried to create a reference with a non-Claim source')
def test_reference_init_with_list(self): r_test = WD.Reference(source_test=[self.ref_1, self.ref_2]) self.assertEqual(r_test.source_test, [self.ref_1, self.ref_2]) self.assertEqual(r_test.source_notest, []) r_notest = WD.Reference(source_notest=[self.ref_1, self.ref_2]) self.assertEqual(r_notest.source_test, []) self.assertEqual(r_notest.source_notest, [self.ref_1, self.ref_2]) r_both = WD.Reference([self.ref_1, self.ref_2], [self.ref_2, self.ref_1]) self.assertEqual(r_both.source_test, [self.ref_1, self.ref_2]) self.assertEqual(r_both.source_notest, [self.ref_2, self.ref_1])
def test_qualifier_equality(self): q = WD.Qualifier('P123', 'foo') q_same = WD.Qualifier('P123', 'foo') q_different = WD.Qualifier('P123', 'bar') self.assertTrue(q == q_same) self.assertFalse(q != q_same) self.assertTrue(q == q) self.assertFalse(q != q) self.assertFalse(q == q_different) self.assertTrue(q != q_different) # Comparison with other classes always gives false, weird but expected self.assertFalse(q == 'foo') self.assertFalse(q != 'foo')
def get_death_place(bot, values): """Get birth place from either birthPlace or birthPlace_P7. @param bot: the instance of the bot calling upon the template @param bot: KulturnavBot @param values: the values extracted using the rules @type values: dict @return: the death place statment @rtype: WD.Statement """ if values.get(u'deathPlace'): return WD.Statement(bot.dbpedia2Wikidata(values[u'deathPlace'])) elif values.get(u'deathPlace_P7'): return WD.Statement(bot.location2Wikidata( values[u'deathPlace_P7']))
def make_ref(self, date): """Make a correctly formatted ref object for claims. Contains 4 parts: * P248: Stated in <the kulturnav dataset> * P577: Publication date <from the document> * P854: Reference url <using the current uuid> * P813: Retrieval date <current date> P854 Should be in source_test (after retroactively fixing older references) but by being in source_notest we ensure that duplicate uuids don't source the statement twice. @param date: The "last modified" time of the document @type date: pywikibot.WbTime @return: the formated reference @rtype WD.Reference """ reference_url = 'http://kulturnav.org/%s' % self.current_uuid ref = WD.Reference( source_test=self.wd.make_simple_claim( 'P248', self.wd.QtoItemPage(self.DATASET_Q)), source_notest=[ self.wd.make_simple_claim('P577', date), self.wd.make_simple_claim('P854', reference_url), self.wd.make_simple_claim('P813', helpers.today_as_WbTime()) ]) return ref
def make_lido_ref(self, lido_data): """ Make a Reference object for the dataset. Contains 4 parts: * P248: Stated in <the Nationalmuseum dataset> * P577: Publication date <from creation date of the document> * P854: Reference url <using the input url> * P813: Retrieval date <current date> """ exit() # P248: Nationalmuseum dataset xml_file = lido_data.get('source_file') date = helpers.today_as_WbTime() pub_date = helpers.iso_to_WbTime(u'2016-09-30') zip_url = u'https://github.com/NationalmuseumSWE/WikidataCollection/' \ u'blob/master/valid_items_transform_1677.tgz' ref = WD.Reference(source_test=[ self.wd.make_simple_claim(u'P854', zip_url), self.wd.make_simple_claim(u'P577', pub_date), self.wd.make_simple_claim(u'P?', xml_file), ], source_notest=self.wd.make_simple_claim( u'P813', date)) return ref
def __init__(self, base_path, new=False, cutoff=None, preview_file=None): """ Initialise the ImporterBot. :param base_path: path to the output directory :param new: whether to also create new items :param cutoff: the number of items to process before stopping. None being interpreted as all. :param preview_file: run in demo mode (create previews rather than live edits) and output the result to this file. """ self.repo = pywikibot.Site().data_repository() self.wd = WdS(self.repo, EDIT_SUMMARY) self.new = new self.cutoff = cutoff if preview_file: self.demo = True self.preview_file = path.join(base_path, preview_file) else: self.demo = False self.preview_data = [] self.set_references() self.place_id_p = 'P3008' # unique identifier property self.country = self.wd.QtoItemPage('Q408') self.states = self.make_states_map() self.settlements = self.make_settlements_map() self.hectares = self.wd.QtoItemPage(helpers.get_unit_q('ha')) self.make_status_and_instance_map() self.place_id_items = helpers.fill_cache_wdqs(self.place_id_p, no_strip=True)
def make_url_ref(self, url, fetch_date, publish_date=None): """Make a Reference object for a url. Contains 3 parts: * P813: Retrieval date * P577: Publication date <from creation date of the document> * P854: Reference url <using the input url> :param url: the source url :param fetch_date: the retrieval date url (iso) :param publish_date: the retrieval date url (iso) :return: WdS.Reference """ date_claims = [] if publish_date: date_claims.append( self.wd.make_simple_claim('P577', helpers.iso_to_WbTime(publish_date))) date_claims.append( self.wd.make_simple_claim('P813', helpers.iso_to_WbTime(fetch_date))) ref = WdS.Reference( source_test=[self.wd.make_simple_claim('P854', url)], source_notest=date_claims) return ref
def set_claim_with_start_and_end(self, prop, target_values, main_key, protoclaims): """ Add a claim with start and end date qualifiers to protoclaims. Requires the value to be resolvable using kulturnav2Wikidata. @param prop: the property of the claim @type prop: str @param target_values: the values for the claim @type target_values: dict|list (of dict)|None @param main_key: the key for the main entry of the target_values dict @type main_key: str @param protoclaims: the dict of claims to add @type protoclaims: dict """ if target_values: target_values = helpers.listify(target_values) claims = [] for val in target_values: claim = WD.Statement(self.kulturnav2Wikidata(val[main_key])) claims.append( helpers.add_start_end_qualifiers(claim, val[u'start'], val[u'end'])) if claims: protoclaims[prop] = claims
def test_statement_none_qualifier(self): s = WD.Statement('foo') s.addQualifier(None) self.assertEquals(s.quals, []) s.addQualifier(None, force=True) s.force = False
def test_reference_repr(self): """Also ensures there is a repr for Claim.""" r = WD.Reference(self.ref_1, self.ref_2) self.assertEqual( repr(r), 'WD.Reference(' 'test: [WD.Claim(P55: foo)], ' 'no_test: [WD.Claim(P55: bar)])')
def test_statement_add_bad_reference_error(self): s = WD.Statement('foo') with self.assertRaises(pwbError) as cm: s.add_reference('foo') self.assertEqual( str(cm.exception), 'add_reference was called with something other ' 'than a Reference object: foo')
def test_statement_init(self): s = WD.Statement('foo') self.assertFalse(s.force) self.assertFalse(s.special) self.assertEquals(s.quals, []) self.assertEquals(s.itis, 'foo') self.assertEquals(s.ref, None)
def make_protoclaims(self, data): """ Construct potential claims for an entry. :param data: dict of data for a single heritage object """ protoclaims = dict() # P17: country protoclaims['P17'] = WdS.Statement(self.country) # P1435: heritage status heritage_type = self.get_heritage_type(data.get('type')) statement = WdS.Statement(self.status[heritage_type]) if data.get('register_date'): statement.addQualifier( WdS.Qualifier('P580', self.parse_date(data.get('register_date')))) protoclaims['P1435'] = statement # P31: class protoclaims['P31'] = WdS.Statement( self.instance_type[data.get('class').lower()]) # P3008: place_id protoclaims[self.place_id_p] = WdS.Statement(data['place_id']) # P131: state protoclaims['P131'] = WdS.Statement( self.get_state(data['state'], data['address'])) # P2046: area if data.get('hectares'): protoclaims['P2046'] = WdS.Statement( pywikibot.WbQuantity(data['hectares'], unit=self.hectares, site=self.wd.repo)) # P969: address if ',' in data['address']: protoclaims['P969'] = WdS.Statement(data['address']) # P276: place protoclaims['P276'] = WdS.Statement( self.get_place(data['state'], data['address'])) # P625: coordinate if data.get('lat') and data.get('lon'): protoclaims['P625'] = self.get_coordinate_statement( data.get('lat'), data.get('lon'), heritage_type) return protoclaims
def test_statement_add_second_reference_error(self): s = WD.Statement('foo').add_reference(self.ref) with self.assertRaises(pwbError) as cm: s.add_reference(self.ref) self.assertEqual( str(cm.exception), 'add_reference was called when the statement already had ' 'a reference assigned to it.')
def unifiedPositionHandler(self, uppdrag, roleMap, entityMap): """Process position based on known mappings. Process positions within a Committee of the Riksdag or a party. param uppdrag: dict param roleMap: str, key within mappings.json param entityMap: str|None, key within mappings.json, skip if None return WD.Statment|None """ # only considered some positions roleCode = uppdrag['roll_kod'] if not self.test_role_code(roleCode, roleMap, uppdrag): return None # expect status = None if uppdrag['status'] is not None: pywikibot.output('Non-none status: %s (%s-%s)' % (uppdrag['status'], uppdrag['typ'], self.current_id)) return None # create statement based on role qNo = self.mappings[roleMap]['Q'][roleCode] statement = WD.Statement(self.wd.QtoItemPage(qNo)) # identify entity if entityMap: entityCode = uppdrag['organ_kod'].upper() if entityCode in self.mappings[entityMap]['Q'].keys(): qNo = self.mappings[entityMap]['Q'][entityCode] qual = WD.Qualifier( P=OF_P, itis=self.wd.QtoItemPage(qNo)) statement.addQualifier(qual) else: pywikibot.output('Unknown entity: %s-%s (%s-%s)' % (entityCode, uppdrag['uppgift'], uppdrag['typ'], self.current_id)) # add standard qualifiers helpers.add_start_end_qualifiers( statement, uppdrag['from'], self.notFuture(uppdrag['tom'])) self.addOrdinal(uppdrag['ordningsnummer'], statement) return statement
def add_inventory_and_collection_claim(self, painting_item, painting_id, painting, uri): """Add an inventory_no, with qualifier, and a collection/P195 claim. This will add the collection qualifier to any matching claim missing it. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @param painting: information object for the painting @type painting: dict @param uri: reference url on nationalmuseum.se @type uri: str """ nationalmuseum_item = self.wd.QtoItemPage(INSTITUTION_Q) collection_p = u'P195' # abort if conflicting info if self.painting_id_prop in painting_item.claims and \ not self.wd.has_claim(self.painting_id_prop, painting_id, painting_item): pywikibot.output( u'%s has conflicting inv. no (%s). Expected %s' % (painting_item, self.painting_id_prop, painting_id)) return # add inventory number with collection self.wd.addNewClaim( self.painting_id_prop, WD.Statement(painting_id).addQualifier(WD.Qualifier( P=collection_p, itis=nationalmuseum_item), force=True), painting_item, self.make_url_reference(uri)) # add collection (or subcollection) subcol = self.prefix_map[painting_id.split(' ')[0]]['subcol'] collection_item = nationalmuseum_item if subcol is not None: collection_item = self.wd.QtoItemPage(subcol) self.wd.addNewClaim(collection_p, WD.Statement(collection_item), painting_item, self.make_europeana_reference(painting))