def claims(self, values):
"""Add protoclaims.
@param values: the values extracted using the rules
@type values: dict
@return: the protoclaims
@rtype: dict PID-WD.Statement pairs
"""
# handle altNames together with names
values[u'entity.name'] = KulturnavBotSMM.prep_names(values)
# bundle type and otherType
values[u'navalVessel.type'] = KulturnavBotSMM.prep_types(values)
protoclaims = {
# operator = Swedish Navy
u'P137': WD.Statement(self.wd.QtoItemPage(self.SWENAVY_Q))
}
# P31 - instance of
# ship class unless a submarine
class_Q = self.SHIPCLASS_Q
if values[u'navalVessel.type'] and \
any(x.endswith(self.SUBMARINETYPE_K)
for x in values[u'navalVessel.type']):
class_Q = self.SUBMARINECLASS_Q
protoclaims[u'P31'] = WD.Statement(self.wd.QtoItemPage(class_Q))
# P279 - subgroup
self.set_subgroup(values, protoclaims)
# P287 - Designer (Constructor)
self.set_constructor(values, protoclaims)
return protoclaims
def set_type_and_class(self, values, protoclaims):
"""Identify type (P31) and class (P289) and add to claims.
Adds the claim to the protoclaims dict.
@param values: the values extracted using the rules
@type values: dict
@param protoclaims: the dict of claims to add
@type protoclaims: dict
"""
if values.get(u'navalVessel.type'):
ship_class = []
ship_type = []
for val in values[u'navalVessel.type']:
item = self.kulturnav2Wikidata(val)
if item:
q = int(item.title()[1:])
if q in self.class_list:
ship_class.append(WD.Statement(item))
elif q in self.type_list:
ship_type.append(WD.Statement(item))
else:
pywikibot.output(u'Q%d not matched as either ship'
u'type or ship class' % q)
if ship_class:
protoclaims[u'P289'] = ship_class
if ship_type:
protoclaims[u'P31'] = ship_type
def set_ship_events(self, values, protoclaims):
"""Identify any events (P793) for a ship then add to claims.
Adds the claim(s) to the protoclaims dict.
Events are only added IFF they have an associated date.
@todo: commissioned: Q14475832
@param values: the values extracted using the rules
@type values: dict
@param protoclaims: the dict of claims to add
@type protoclaims: dict
"""
events = []
# built: Q474200
event = WD.Statement(self.wd.QtoItemPage('Q474200'))
if self.set_date_qualifier(values, 'built', event, prop=helpers.END_P):
self.set_location_qualifier(values, 'built', event)
# u'built.shipyard'
events.append(event)
# launched: Q596643
event = WD.Statement(self.wd.QtoItemPage('Q596643'))
if self.set_date_qualifier(values, 'launched', event):
# u'launched.shipyard'
events.append(event)
# decommissioned: Q7497952
event = WD.Statement(self.wd.QtoItemPage('Q7497952'))
if self.set_date_qualifier(values, 'decommissioned', event):
events.append(event)
# set all events
if events:
protoclaims[u'P793'] = events
def set_creator(self,
target_item,
reference,
creator_q=None,
related_info=None):
"""Set a creator/P170 claim for a creator or creator combo.
Allows for simple claims as well as more complex
"in the manner of" etc.
@param target_item: item to which claim is added
@type target_item: pywikibot.ItemPage
@param reference: the reference for the statment
@type reference: WD.Reference
@param related_info: related info as a dict with P/itis pairs
@type related_info: dict
@param creator_q: the Q-id of the creator
@type creator_q: str
"""
creator_q = creator_q or ANON_Q
creator_statement = WD.Statement(self.wd.QtoItemPage(creator_q))
# set any related qualifiers
if related_info:
creator_statement.addQualifier(
WD.Qualifier(P=related_info['P'], itis=related_info['itis']))
# set claim
self.wd.addNewClaim(u'P170', creator_statement, target_item, reference)
def add_start_end_qualifiers(statement, startVal, endVal):
"""
Add start/end qualifiers to a statement if non-None, or return None.
@param statement: The statement to decorate
@type statement: WD.Statement
@param startVal: An ISO date string for the starting point
@type startVal: basestring or None
@param endVal: An ISO date string for the end point
@type endVal: basestring or None
@return: A statement decorated with start/end qualifiers
@rtype: WD.Statement, or None
"""
if not isinstance(statement, WD.Statement):
raise pywikibot.Error('Non-statement recieved: %s' % statement)
if statement.isNone():
return None
# add qualifiers
quals = []
if startVal:
quals.append(
WD.Qualifier(
P=START_P,
itis=iso_to_WbTime(startVal)))
if endVal:
quals.append(
WD.Qualifier(
P=END_P,
itis=iso_to_WbTime(endVal)))
for q in quals:
statement.addQualifier(q)
return statement
def test_statement_equality_qualifier_order(self):
s_1 = WD.Statement('foo')
s_2 = WD.Statement('foo')
s_3 = WD.Statement('foo')
s_1.addQualifier(self.q_1).addQualifier(self.q_2)
s_2.addQualifier(self.q_2).addQualifier(self.q_1)
s_3.addQualifier(self.q_1)
self.assertEquals(s_1, s_2)
self.assertNotEquals(s_1, s_3)
def setUp(self):
wikidata = Site('test', 'wikidata')
self.q_1 = WD.Qualifier('P123', 'foo')
self.q_2 = WD.Qualifier('P123', 'bar')
claim = Claim(wikidata, 'P55')
claim.setTarget('foo')
self.ref = WD.Reference(source_test=[
claim,
])
def test_reference_get_all_sources(self):
r_test = WD.Reference(source_test=self.ref_1)
self.assertEqual(r_test.get_all_sources(), [self.ref_1])
r_notest = WD.Reference(source_notest=self.ref_1)
self.assertEqual(r_notest.get_all_sources(), [self.ref_1])
r_both = WD.Reference(self.ref_1, self.ref_2)
self.assertEqual(r_both.get_all_sources(), [self.ref_1, self.ref_2])
def test_qualifier_hash(self):
q_1 = WD.Qualifier('P123', 'foo')
q_2 = WD.Qualifier('P124', 'foo')
q_3 = WD.Qualifier('P123', 'bar')
q_same = WD.Qualifier('P123', 'foo')
self.assertEqual(q_1, q_same)
self.assertNotEqual(q_1, q_2)
self.assertNotEqual(q_1, q_3)
self.assertNotEqual(q_2, q_3)
def test_statement_inequality(self):
s = WD.Statement('novalue')
s_special = WD.Statement('novalue', special=True)
s_force = WD.Statement('novalue')
s_force.force = True
self.assertNotEquals(s, s_special)
self.assertNotEquals(s, s_force)
self.assertNotEquals(s_force, s_special)
def test_statement_init_special(self):
self.assertTrue(WD.Statement('somevalue', special=True).special)
self.assertTrue(WD.Statement('novalue', special=True).special)
with self.assertRaises(pwbError) as cm:
WD.Statement('foo', special=True)
self.assertEqual(
str(cm.exception),
'You tried to create a special statement with a non-allowed '
'snakvalue: foo')
def test_statement_equality(self):
s = WD.Statement('foo')
s_same = WD.Statement('foo')
s_different = WD.Statement('bar')
self.assertEquals(s, s)
self.assertEquals(s, s_same)
self.assertNotEquals(s, s_different)
# Comparison with other classes always gives false, weird but expected
self.assertFalse(s == 'foo')
self.assertFalse(s != 'foo')
def test_reference_init_single_claim_gives_list(self):
r_test = WD.Reference(source_test=self.ref_1)
self.assertEqual(r_test.source_test, [self.ref_1])
self.assertEqual(r_test.source_notest, [])
r_notest = WD.Reference(source_notest=self.ref_1)
self.assertEqual(r_notest.source_test, [])
self.assertEqual(r_notest.source_notest, [self.ref_1])
r_both = WD.Reference(self.ref_1, self.ref_2)
self.assertEqual(r_both.source_test, [self.ref_1])
self.assertEqual(r_both.source_notest, [self.ref_2])
def test_reference_init_non_claim_error(self):
with self.assertRaises(pwbError) as cm:
WD.Reference(source_test='foo')
self.assertEqual(
str(cm.exception),
'You tried to create a reference with a non-Claim source')
with self.assertRaises(pwbError) as cm:
WD.Reference(source_notest='foo')
self.assertEqual(
str(cm.exception),
'You tried to create a reference with a non-Claim source')
def test_reference_init_with_list(self):
r_test = WD.Reference(source_test=[self.ref_1, self.ref_2])
self.assertEqual(r_test.source_test, [self.ref_1, self.ref_2])
self.assertEqual(r_test.source_notest, [])
r_notest = WD.Reference(source_notest=[self.ref_1, self.ref_2])
self.assertEqual(r_notest.source_test, [])
self.assertEqual(r_notest.source_notest, [self.ref_1, self.ref_2])
r_both = WD.Reference([self.ref_1, self.ref_2],
[self.ref_2, self.ref_1])
self.assertEqual(r_both.source_test, [self.ref_1, self.ref_2])
self.assertEqual(r_both.source_notest, [self.ref_2, self.ref_1])
def test_qualifier_equality(self):
q = WD.Qualifier('P123', 'foo')
q_same = WD.Qualifier('P123', 'foo')
q_different = WD.Qualifier('P123', 'bar')
self.assertTrue(q == q_same)
self.assertFalse(q != q_same)
self.assertTrue(q == q)
self.assertFalse(q != q)
self.assertFalse(q == q_different)
self.assertTrue(q != q_different)
# Comparison with other classes always gives false, weird but expected
self.assertFalse(q == 'foo')
self.assertFalse(q != 'foo')
def get_death_place(bot, values):
"""Get birth place from either birthPlace or birthPlace_P7.
@param bot: the instance of the bot calling upon the template
@param bot: KulturnavBot
@param values: the values extracted using the rules
@type values: dict
@return: the death place statment
@rtype: WD.Statement
"""
if values.get(u'deathPlace'):
return WD.Statement(bot.dbpedia2Wikidata(values[u'deathPlace']))
elif values.get(u'deathPlace_P7'):
return WD.Statement(bot.location2Wikidata(
values[u'deathPlace_P7']))
def make_ref(self, date):
"""Make a correctly formatted ref object for claims.
Contains 4 parts:
* P248: Stated in <the kulturnav dataset>
* P577: Publication date <from the document>
* P854: Reference url <using the current uuid>
* P813: Retrieval date <current date>
P854
Should be in source_test (after retroactively fixing older references)
but by being in source_notest we ensure that duplicate uuids don't
source the statement twice.
@param date: The "last modified" time of the document
@type date: pywikibot.WbTime
@return: the formated reference
@rtype WD.Reference
"""
reference_url = 'http://kulturnav.org/%s' % self.current_uuid
ref = WD.Reference(
source_test=self.wd.make_simple_claim(
'P248', self.wd.QtoItemPage(self.DATASET_Q)),
source_notest=[
self.wd.make_simple_claim('P577', date),
self.wd.make_simple_claim('P854', reference_url),
self.wd.make_simple_claim('P813', helpers.today_as_WbTime())
])
return ref
def make_lido_ref(self, lido_data):
"""
Make a Reference object for the dataset.
Contains 4 parts:
* P248: Stated in <the Nationalmuseum dataset>
* P577: Publication date <from creation date of the document>
* P854: Reference url <using the input url>
* P813: Retrieval date <current date>
"""
exit()
# P248: Nationalmuseum dataset
xml_file = lido_data.get('source_file')
date = helpers.today_as_WbTime()
pub_date = helpers.iso_to_WbTime(u'2016-09-30')
zip_url = u'https://github.com/NationalmuseumSWE/WikidataCollection/' \
u'blob/master/valid_items_transform_1677.tgz'
ref = WD.Reference(source_test=[
self.wd.make_simple_claim(u'P854', zip_url),
self.wd.make_simple_claim(u'P577', pub_date),
self.wd.make_simple_claim(u'P?', xml_file),
],
source_notest=self.wd.make_simple_claim(
u'P813', date))
return ref
def __init__(self, base_path, new=False, cutoff=None, preview_file=None):
"""
Initialise the ImporterBot.
:param base_path: path to the output directory
:param new: whether to also create new items
:param cutoff: the number of items to process before stopping. None
being interpreted as all.
:param preview_file: run in demo mode (create previews rather than
live edits) and output the result to this file.
"""
self.repo = pywikibot.Site().data_repository()
self.wd = WdS(self.repo, EDIT_SUMMARY)
self.new = new
self.cutoff = cutoff
if preview_file:
self.demo = True
self.preview_file = path.join(base_path, preview_file)
else:
self.demo = False
self.preview_data = []
self.set_references()
self.place_id_p = 'P3008' # unique identifier property
self.country = self.wd.QtoItemPage('Q408')
self.states = self.make_states_map()
self.settlements = self.make_settlements_map()
self.hectares = self.wd.QtoItemPage(helpers.get_unit_q('ha'))
self.make_status_and_instance_map()
self.place_id_items = helpers.fill_cache_wdqs(self.place_id_p,
no_strip=True)
def make_url_ref(self, url, fetch_date, publish_date=None):
"""Make a Reference object for a url.
Contains 3 parts:
* P813: Retrieval date
* P577: Publication date <from creation date of the document>
* P854: Reference url <using the input url>
:param url: the source url
:param fetch_date: the retrieval date url (iso)
:param publish_date: the retrieval date url (iso)
:return: WdS.Reference
"""
date_claims = []
if publish_date:
date_claims.append(
self.wd.make_simple_claim('P577',
helpers.iso_to_WbTime(publish_date)))
date_claims.append(
self.wd.make_simple_claim('P813',
helpers.iso_to_WbTime(fetch_date)))
ref = WdS.Reference(
source_test=[self.wd.make_simple_claim('P854', url)],
source_notest=date_claims)
return ref
def set_claim_with_start_and_end(self, prop, target_values, main_key,
protoclaims):
"""
Add a claim with start and end date qualifiers to protoclaims.
Requires the value to be resolvable using kulturnav2Wikidata.
@param prop: the property of the claim
@type prop: str
@param target_values: the values for the claim
@type target_values: dict|list (of dict)|None
@param main_key: the key for the main entry of the target_values dict
@type main_key: str
@param protoclaims: the dict of claims to add
@type protoclaims: dict
"""
if target_values:
target_values = helpers.listify(target_values)
claims = []
for val in target_values:
claim = WD.Statement(self.kulturnav2Wikidata(val[main_key]))
claims.append(
helpers.add_start_end_qualifiers(claim, val[u'start'],
val[u'end']))
if claims:
protoclaims[prop] = claims
def test_statement_none_qualifier(self):
s = WD.Statement('foo')
s.addQualifier(None)
self.assertEquals(s.quals, [])
s.addQualifier(None, force=True)
s.force = False
def test_reference_repr(self):
"""Also ensures there is a repr for Claim."""
r = WD.Reference(self.ref_1, self.ref_2)
self.assertEqual(
repr(r), 'WD.Reference('
'test: [WD.Claim(P55: foo)], '
'no_test: [WD.Claim(P55: bar)])')
def test_statement_add_bad_reference_error(self):
s = WD.Statement('foo')
with self.assertRaises(pwbError) as cm:
s.add_reference('foo')
self.assertEqual(
str(cm.exception), 'add_reference was called with something other '
'than a Reference object: foo')
def test_statement_init(self):
s = WD.Statement('foo')
self.assertFalse(s.force)
self.assertFalse(s.special)
self.assertEquals(s.quals, [])
self.assertEquals(s.itis, 'foo')
self.assertEquals(s.ref, None)
def make_protoclaims(self, data):
"""
Construct potential claims for an entry.
:param data: dict of data for a single heritage object
"""
protoclaims = dict()
# P17: country
protoclaims['P17'] = WdS.Statement(self.country)
# P1435: heritage status
heritage_type = self.get_heritage_type(data.get('type'))
statement = WdS.Statement(self.status[heritage_type])
if data.get('register_date'):
statement.addQualifier(
WdS.Qualifier('P580',
self.parse_date(data.get('register_date'))))
protoclaims['P1435'] = statement
# P31: class
protoclaims['P31'] = WdS.Statement(
self.instance_type[data.get('class').lower()])
# P3008: place_id
protoclaims[self.place_id_p] = WdS.Statement(data['place_id'])
# P131: state
protoclaims['P131'] = WdS.Statement(
self.get_state(data['state'], data['address']))
# P2046: area
if data.get('hectares'):
protoclaims['P2046'] = WdS.Statement(
pywikibot.WbQuantity(data['hectares'],
unit=self.hectares,
site=self.wd.repo))
# P969: address
if ',' in data['address']:
protoclaims['P969'] = WdS.Statement(data['address'])
# P276: place
protoclaims['P276'] = WdS.Statement(
self.get_place(data['state'], data['address']))
# P625: coordinate
if data.get('lat') and data.get('lon'):
protoclaims['P625'] = self.get_coordinate_statement(
data.get('lat'), data.get('lon'), heritage_type)
return protoclaims
def test_statement_add_second_reference_error(self):
s = WD.Statement('foo').add_reference(self.ref)
with self.assertRaises(pwbError) as cm:
s.add_reference(self.ref)
self.assertEqual(
str(cm.exception),
'add_reference was called when the statement already had '
'a reference assigned to it.')
def unifiedPositionHandler(self, uppdrag, roleMap, entityMap):
"""Process position based on known mappings.
Process positions within a Committee of the Riksdag or a party.
param uppdrag: dict
param roleMap: str, key within mappings.json
param entityMap: str|None, key within mappings.json, skip if None
return WD.Statment|None
"""
# only considered some positions
roleCode = uppdrag['roll_kod']
if not self.test_role_code(roleCode, roleMap, uppdrag):
return None
# expect status = None
if uppdrag['status'] is not None:
pywikibot.output('Non-none status: %s (%s-%s)' %
(uppdrag['status'], uppdrag['typ'],
self.current_id))
return None
# create statement based on role
qNo = self.mappings[roleMap]['Q'][roleCode]
statement = WD.Statement(self.wd.QtoItemPage(qNo))
# identify entity
if entityMap:
entityCode = uppdrag['organ_kod'].upper()
if entityCode in self.mappings[entityMap]['Q'].keys():
qNo = self.mappings[entityMap]['Q'][entityCode]
qual = WD.Qualifier(
P=OF_P,
itis=self.wd.QtoItemPage(qNo))
statement.addQualifier(qual)
else:
pywikibot.output('Unknown entity: %s-%s (%s-%s)' %
(entityCode, uppdrag['uppgift'],
uppdrag['typ'], self.current_id))
# add standard qualifiers
helpers.add_start_end_qualifiers(
statement, uppdrag['from'], self.notFuture(uppdrag['tom']))
self.addOrdinal(uppdrag['ordningsnummer'], statement)
return statement
def add_inventory_and_collection_claim(self, painting_item, painting_id,
painting, uri):
"""Add an inventory_no, with qualifier, and a collection/P195 claim.
This will add the collection qualifier to any matching
claim missing it.
@param painting_item: item to which claim is added
@type painting_item: pywikibot.ItemPage
@param painting_id: the common (older) id of the painting in the
Nationalmuseum collection
@type painting_id: str
@param painting: information object for the painting
@type painting: dict
@param uri: reference url on nationalmuseum.se
@type uri: str
"""
nationalmuseum_item = self.wd.QtoItemPage(INSTITUTION_Q)
collection_p = u'P195'
# abort if conflicting info
if self.painting_id_prop in painting_item.claims and \
not self.wd.has_claim(self.painting_id_prop, painting_id,
painting_item):
pywikibot.output(
u'%s has conflicting inv. no (%s). Expected %s' %
(painting_item, self.painting_id_prop, painting_id))
return
# add inventory number with collection
self.wd.addNewClaim(
self.painting_id_prop,
WD.Statement(painting_id).addQualifier(WD.Qualifier(
P=collection_p, itis=nationalmuseum_item),
force=True), painting_item,
self.make_url_reference(uri))
# add collection (or subcollection)
subcol = self.prefix_map[painting_id.split(' ')[0]]['subcol']
collection_item = nationalmuseum_item
if subcol is not None:
collection_item = self.wd.QtoItemPage(subcol)
self.wd.addNewClaim(collection_p, WD.Statement(collection_item),
painting_item,
self.make_europeana_reference(painting))