def add_start_end_qualifiers(statement, startVal, endVal): """ Add start/end qualifiers to a statement if non-None, or return None. @param statement: The statement to decorate @type statement: WD.Statement @param startVal: An ISO date string for the starting point @type startVal: basestring or None @param endVal: An ISO date string for the end point @type endVal: basestring or None @return: A statement decorated with start/end qualifiers @rtype: WD.Statement, or None """ if not isinstance(statement, WD.Statement): raise pywikibot.Error('Non-statement recieved: %s' % statement) if statement.isNone(): return None # add qualifiers quals = [] if startVal: quals.append( WD.Qualifier( P=START_P, itis=iso_to_WbTime(startVal))) if endVal: quals.append( WD.Qualifier( P=END_P, itis=iso_to_WbTime(endVal))) for q in quals: statement.addQualifier(q) return statement
def setUp(self): wikidata = Site('test', 'wikidata') self.q_1 = WD.Qualifier('P123', 'foo') self.q_2 = WD.Qualifier('P123', 'bar') claim = Claim(wikidata, 'P55') claim.setTarget('foo') self.ref = WD.Reference(source_test=[ claim, ])
def test_qualifier_hash(self): q_1 = WD.Qualifier('P123', 'foo') q_2 = WD.Qualifier('P124', 'foo') q_3 = WD.Qualifier('P123', 'bar') q_same = WD.Qualifier('P123', 'foo') self.assertEqual(q_1, q_same) self.assertNotEqual(q_1, q_2) self.assertNotEqual(q_1, q_3) self.assertNotEqual(q_2, q_3)
def test_qualifier_equality(self): q = WD.Qualifier('P123', 'foo') q_same = WD.Qualifier('P123', 'foo') q_different = WD.Qualifier('P123', 'bar') self.assertTrue(q == q_same) self.assertFalse(q != q_same) self.assertTrue(q == q) self.assertFalse(q != q) self.assertFalse(q == q_different) self.assertTrue(q != q_different) # Comparison with other classes always gives false, weird but expected self.assertFalse(q == 'foo') self.assertFalse(q != 'foo')
def make_base_protoclaims(self, values, protoclaims): """Construct the protoclaims common for all KulturnavBots. Adds the claim to the protoclaims dict. @param values: the values extracted using the rules @type values: dict @param protoclaims: the dict of claims to add @type protoclaims: dict """ # kulturnav protoclaim incl. qualifier protoclaims[u'P%s' % self.KULTURNAV_ID_P] = \ WD.Statement(values[u'identifier']).addQualifier( WD.Qualifier( P=self.CATALOG_P, itis=self.wd.QtoItemPage(self.DATASET_Q)), force=True) # authority control protoclaims if values.get(u'libris-id'): protoclaims[u'P906'] = WD.Statement(values[u'libris-id']) if values.get(u'viaf-id'): protoclaims[u'P214'] = WD.Statement(values[u'viaf-id']) if values.get(u'getty_aat'): protoclaims[u'P1014'] = WD.Statement(values[u'getty_aat']) if values.get(u'ulan'): protoclaims[u'P245'] = WD.Statement(values[u'ulan'])
def set_creator(self, target_item, reference, creator_q=None, related_info=None): """Set a creator/P170 claim for a creator or creator combo. Allows for simple claims as well as more complex "in the manner of" etc. @param target_item: item to which claim is added @type target_item: pywikibot.ItemPage @param reference: the reference for the statment @type reference: WD.Reference @param related_info: related info as a dict with P/itis pairs @type related_info: dict @param creator_q: the Q-id of the creator @type creator_q: str """ creator_q = creator_q or ANON_Q creator_statement = WD.Statement(self.wd.QtoItemPage(creator_q)) # set any related qualifiers if related_info: creator_statement.addQualifier( WD.Qualifier(P=related_info['P'], itis=related_info['itis'])) # set claim self.wd.addNewClaim(u'P170', creator_statement, target_item, reference)
def make_protoclaims(self, data): """ Construct potential claims for an entry. :param data: dict of data for a single heritage object """ protoclaims = dict() # P17: country protoclaims['P17'] = WdS.Statement(self.country) # P1435: heritage status heritage_type = self.get_heritage_type(data.get('type')) statement = WdS.Statement(self.status[heritage_type]) if data.get('register_date'): statement.addQualifier( WdS.Qualifier('P580', self.parse_date(data.get('register_date')))) protoclaims['P1435'] = statement # P31: class protoclaims['P31'] = WdS.Statement( self.instance_type[data.get('class').lower()]) # P3008: place_id protoclaims[self.place_id_p] = WdS.Statement(data['place_id']) # P131: state protoclaims['P131'] = WdS.Statement( self.get_state(data['state'], data['address'])) # P2046: area if data.get('hectares'): protoclaims['P2046'] = WdS.Statement( pywikibot.WbQuantity(data['hectares'], unit=self.hectares, site=self.wd.repo)) # P969: address if ',' in data['address']: protoclaims['P969'] = WdS.Statement(data['address']) # P276: place protoclaims['P276'] = WdS.Statement( self.get_place(data['state'], data['address'])) # P625: coordinate if data.get('lat') and data.get('lon'): protoclaims['P625'] = self.get_coordinate_statement( data.get('lat'), data.get('lon'), heritage_type) return protoclaims
def addOrdinal(self, value, statement): """Add an ordinal qualifier. Adds the ordinal as a qualifier if it is non-zero param value: ordinal param statment: statment to add qualifer to """ if value != '0': qual = WD.Qualifier( P=ORDINAL_P, itis=int(value)) statement.addQualifier(qual)
def unifiedPositionHandler(self, uppdrag, roleMap, entityMap): """Process position based on known mappings. Process positions within a Committee of the Riksdag or a party. param uppdrag: dict param roleMap: str, key within mappings.json param entityMap: str|None, key within mappings.json, skip if None return WD.Statment|None """ # only considered some positions roleCode = uppdrag['roll_kod'] if not self.test_role_code(roleCode, roleMap, uppdrag): return None # expect status = None if uppdrag['status'] is not None: pywikibot.output('Non-none status: %s (%s-%s)' % (uppdrag['status'], uppdrag['typ'], self.current_id)) return None # create statement based on role qNo = self.mappings[roleMap]['Q'][roleCode] statement = WD.Statement(self.wd.QtoItemPage(qNo)) # identify entity if entityMap: entityCode = uppdrag['organ_kod'].upper() if entityCode in self.mappings[entityMap]['Q'].keys(): qNo = self.mappings[entityMap]['Q'][entityCode] qual = WD.Qualifier( P=OF_P, itis=self.wd.QtoItemPage(qNo)) statement.addQualifier(qual) else: pywikibot.output('Unknown entity: %s-%s (%s-%s)' % (entityCode, uppdrag['uppgift'], uppdrag['typ'], self.current_id)) # add standard qualifiers helpers.add_start_end_qualifiers( statement, uppdrag['from'], self.notFuture(uppdrag['tom'])) self.addOrdinal(uppdrag['ordningsnummer'], statement) return statement
def add_inventory_and_collection_claim(self, painting_item, painting_id, painting, uri): """Add an inventory_no, with qualifier, and a collection/P195 claim. This will add the collection qualifier to any matching claim missing it. @param painting_item: item to which claim is added @type painting_item: pywikibot.ItemPage @param painting_id: the common (older) id of the painting in the Nationalmuseum collection @type painting_id: str @param painting: information object for the painting @type painting: dict @param uri: reference url on nationalmuseum.se @type uri: str """ nationalmuseum_item = self.wd.QtoItemPage(INSTITUTION_Q) collection_p = u'P195' # abort if conflicting info if self.painting_id_prop in painting_item.claims and \ not self.wd.has_claim(self.painting_id_prop, painting_id, painting_item): pywikibot.output( u'%s has conflicting inv. no (%s). Expected %s' % (painting_item, self.painting_id_prop, painting_id)) return # add inventory number with collection self.wd.addNewClaim( self.painting_id_prop, WD.Statement(painting_id).addQualifier(WD.Qualifier( P=collection_p, itis=nationalmuseum_item), force=True), painting_item, self.make_url_reference(uri)) # add collection (or subcollection) subcol = self.prefix_map[painting_id.split(' ')[0]]['subcol'] collection_item = nationalmuseum_item if subcol is not None: collection_item = self.wd.QtoItemPage(subcol) self.wd.addNewClaim(collection_p, WD.Statement(collection_item), painting_item, self.make_europeana_reference(painting))
def set_location_qualifier(self, values, key, statement): """Add a location (P279) qualifier to a statement. @param values: the values extracted using the rules @type values: dict @param key: the key to which the location is associated e.g. built for built.location @type key: str @param statement: statment to add qualifier to @type statement: WD.Statement @return: if qualifier was found @rtype: bool """ location_key = u'%s.location' % key if not values[location_key]: return False statement.addQualifier( WD.Qualifier(P=self.PLACE_P, itis=self.location2Wikidata(values[location_key]))) return True
def set_date_qualifier(self, values, key, statement, prop=None): """Add a date qualifier to a statement. @param values: the values extracted using the rules @type values: dict @param key: the key to which the location is associated e.g. built for built.location @type key: str @param statement: statment to add qualifier to @type statement: WD.Statement @param prop: the property to use, defaults to self.TIME_P/P585 @type prop: str @return: if qualifier was found @rtype: bool """ prop = prop or self.TIME_P date_key = u'%s.date' % key if not values[date_key]: return False statement.addQualifier( WD.Qualifier(P=prop, itis=helpers.iso_to_WbTime(values[date_key]))) return True
def test_qualifier_repr(self): q = WD.Qualifier('P123', 'foo') self.assertEqual(repr(q), 'WD.Qualifier(P123, foo)')
class WikidataItem(object): def __init__(self, db_row_dict, repository, data_files, existing, caches): self.repo = repository self.existing = existing self.wdstuff = WDS(self.repo) self.raw_data = db_row_dict self.caches = caches self.problem_report = {} self.props = data_files["properties"] self.construct_wd_item() self.problem_report = {} def get_caches(self): return self.caches def make_q_item(self, qnumber): return self.wdstuff.QtoItemPage(qnumber) def make_pywikibot_item(self, value): val_item = None if isinstance(value, list) and len(value) == 1: value = value[0] if utils.string_is_q_item(value): val_item = self.make_q_item(value) elif value == "novalue": val_item = value elif isinstance(value, dict) and 'monolingual_value' in value: text = value['monolingual_value'] language = value['lang'] val_item = pywikibot.WbMonolingualText(text=text, language=language) elif isinstance(value, dict) and 'quantity_value' in value: number = value['quantity_value'] if 'unit' in value: unit = self.wdstuff.QtoItemPage(value["unit"]) else: unit = None val_item = pywikibot.WbQuantity(amount=number, unit=unit, site=self.repo) elif isinstance(value, dict) and 'date_value' in value: date_dict = value["date_value"] val_item = pywikibot.WbTime(year=date_dict.get("year"), month=date_dict.get("month"), day=date_dict.get("day")) elif value == "novalue": # raise NotImplementedError # implement Error print("Status: novalue will be added here") else: val_item = value return val_item def make_statement(self, value): if value in ['somevalue', 'novalue']: special = True else: special = False return self.wdstuff.Statement(value, special=special) def make_qualifier_applies_to(self, value): prop_item = self.props["applies_to_part"] target_item = self.wdstuff.QtoItemPage(value) return self.wdstuff.Qualifier(prop_item, target_item) def add_statement(self, prop_name, value, quals=None, ref=None): base = self.wd_item["statements"] prop = self.props[prop_name] if quals is None: quals = [] wd_claim = self.make_pywikibot_item(value) statement = self.make_statement(wd_claim) for qual in helpers.listify(quals): statement.addQualifier(qual) base.append({"prop": prop, "value": statement, "ref": ref}) def make_stated_in_ref(self, value, pub_date=None, ref_url=None, retrieved_date=None): item_prop = self.props["stated_in"] published_prop = self.props["publication_date"] published_claim = None if pub_date: pub_date = utils.date_to_dict(pub_date, "%Y-%m-%d") timestamp = self.make_pywikibot_item({"date_value": pub_date}) published_claim = self.wdstuff.make_simple_claim( published_prop, timestamp) source_item = self.wdstuff.QtoItemPage(value) source_claim = self.wdstuff.make_simple_claim(item_prop, source_item) if ref_url and retrieved_date: ref_url_prop = self.props["reference_url"] retrieved_date_prop = self.props["retrieved"] retrieved_date = utils.date_to_dict(retrieved_date, "%Y-%m-%d") retrieved_date = self.make_pywikibot_item( {"date_value": retrieved_date}) ref_url_claim = self.wdstuff.make_simple_claim( ref_url_prop, ref_url) retrieved_on_claim = self.wdstuff.make_simple_claim( retrieved_date_prop, retrieved_date) if published_claim: ref = self.wdstuff.Reference( source_test=[source_claim, ref_url_claim], source_notest=[published_claim, retrieved_on_claim]) else: ref = self.wdstuff.Reference( source_test=[source_claim, ref_url_claim], source_notest=[retrieved_on_claim]) else: ref = self.wdstuff.Reference(source_test=[source_claim], source_notest=published_claim) return ref def associate_wd_item(self, wd_item): if wd_item is not None: self.wd_item["wd-item"] = wd_item def set_upload(self, booln): self.wd_item["upload"] = booln def add_label(self, language, text): base = self.wd_item["labels"] base.append({"language": language, "value": text}) def add_description(self, language, text): base = self.wd_item["descriptions"] base.append({"language": language, "value": text}) def add_to_report(self, key_name, raw_data, id_no, prop_name=None): """ Add data to problem report json. Check if item has an associated Q-number, and if that's the case and it's missing in the report, add it to the report automatically. Optionally, assign a Property ID that the data should have been used as a value for. :param key_name: name of the field containing the problematic data, e.g. the header of the column :type key_name: string :param raw_data: the data that we failed to process :type raw_data: string :param id_no: unique id assigned to item, e.g. url :type id_no: string :param prop_name: name of the property, as stated in the props library file :type prop_name: string """ prop = None if prop_name: if prop_name.startswith('_'): prop = prop_name else: prop = self.props.get(prop_name) self.problem_report[key_name] = {"value": raw_data, "target": prop} if "wd-item" not in self.problem_report: if self.wd_item["wd-item"] is not None: self.problem_report["Q"] = self.wd_item["wd-item"] else: self.problem_report["Q"] = "" self.problem_report["url"] = id_no def print_report(self): """Print the problem report on screen.""" print( json.dumps(self.problem_report, sort_keys=True, indent=4, ensure_ascii=False, default=utils.datetime_convert)) def get_report(self): """Retrieve the problem report.""" return self.problem_report def construct_wd_item(self): self.wd_item = {} self.wd_item["upload"] = True self.wd_item["statements"] = [] self.wd_item["labels"] = [] self.wd_item["descriptions"] = [] self.wd_item["wd-item"] = None
class WikidataItem(object): """Basic data object for upload to Wikidata.""" def __init__(self, db_row_dict, repository, data_files, existing): """ Initialize the data object. :param db_row_dict: raw data from the data source :type db_row_dict: string :param repository: data repository (Wikidata site) :type repository: site instance :param data_files: dict of various mapping files :type data_files: dictionary :param existing: WD items that already have an unique id :type existing: dictionary """ self.repo = repository self.existing = existing self.wdstuff = WDS(self.repo) self.raw_data = db_row_dict self.props = data_files["properties"] self.items = data_files["items"] self.construct_wd_item() self.problem_report = {} def make_q_item(self, qnumber): """ Create a regular Wikidata ItemPage. :param qnumber: Q-item that we want to get an ItemPage of :type qnumber: string :return: an ItemPage for pywikibot """ return self.wdstuff.QtoItemPage(qnumber) def make_pywikibot_item(self, value): """ Create a statement in pywikibot-ready format. The statement can be either: * a string (value is string) * an item (value is Q-string) * an amount with or without unit (value is dic) :param value: the content of the item :type value: it can be a string or a dictionary, see above. :return: a pywikibot item of the type determined by the input data, either ItemPage or Quantity or string. """ val_item = None if isinstance(value, list) and len(value) == 1: value = value[0] if utils.string_is_q_item(value): val_item = self.make_q_item(value) elif value == "novalue": val_item = value elif isinstance(value, dict) and 'quantity_value' in value: number = value['quantity_value'] if 'unit' in value: unit = self.wdstuff.QtoItemPage(value["unit"]) else: unit = None val_item = pywikibot.WbQuantity( amount=number, unit=unit, site=self.repo) elif isinstance(value, dict) and 'date_value' in value: date_dict = value["date_value"] val_item = pywikibot.WbTime(year=date_dict["year"], month=date_dict["month"], day=date_dict["day"]) elif value == "novalue": # raise NotImplementedError # implement Error print("Status: novalue will be added here") else: val_item = value return val_item def make_statement(self, value): """ Create a Wikidatastuff statement. Supports the special data types 'somevalue' and 'novalue'. :prop value: the content of the statement :type value: pywikibot item :return: a wikidatastuff statement """ if value in ['somevalue', 'novalue']: special = True else: special = False return self.wdstuff.Statement(value, special=special) def make_qualifier_applies_to(self, value): """ Create a qualifier to a statement with type 'applies to part'. :param value: Q-item that this applies to :type value: string :return: a wikidatastuff Qualifier """ prop_item = self.props["applies_to_part"] target_item = self.wdstuff.QtoItemPage(value) return self.wdstuff.Qualifier(prop_item, target_item) def add_statement(self, prop_name, value, quals=None, ref=None): """ Add a statement to the data object. :param prop_name: P-item representing property :type prop_name: string :param value: content of the statement :type value: it can be a string representing a Q-item or a dictionary of an amount :param quals: possibly qualifier items :type quals: a wikidatastuff Qualifier item, or a list of them :param ref: reference item :type ref: a wikidatastuff Reference item """ base = self.wd_item["statements"] prop = self.props[prop_name] if quals is None: quals = [] wd_claim = self.make_pywikibot_item(value) statement = self.make_statement(wd_claim) for qual in helpers.listify(quals): statement.addQualifier(qual) base.append({"prop": prop, "value": statement, "ref": ref}) def make_stated_in_ref(self, value, pub_date, ref_url=None, retrieved_date=None): """ Make a reference object of type 'stated in'. :param value: Q-item where sth is stated :type value: string :param pub_date: timestamp in format "1999-09-31" :type pub_date: string :param ref_url: optionally a reference url :type ref_url: string :param retrieved_date: timestamp in format "1999-09-31" :type retrieved_date: string :return: a wikidatastuff Reference item """ item_prop = self.props["stated_in"] published_prop = self.props["publication_date"] pub_date = utils.date_to_dict(pub_date, "%Y-%m-%d") timestamp = self.make_pywikibot_item({"date_value": pub_date}) published_claim = self.wdstuff.make_simple_claim( published_prop, timestamp) source_item = self.wdstuff.QtoItemPage(value) source_claim = self.wdstuff.make_simple_claim(item_prop, source_item) if ref_url and retrieved_date: ref_url_prop = self.props["reference_url"] retrieved_date_prop = self.props["retrieved"] retrieved_date = utils.date_to_dict(retrieved_date, "%Y-%m-%d") retrieved_date = self.make_pywikibot_item( {"date_value": retrieved_date}) ref_url_claim = self.wdstuff.make_simple_claim( ref_url_prop, ref_url) retrieved_on_claim = self.wdstuff.make_simple_claim( retrieved_date_prop, retrieved_date) ref = self.wdstuff.Reference( source_test=[source_claim, ref_url_claim], source_notest=[published_claim, retrieved_on_claim]) else: ref = self.wdstuff.Reference( source_test=[source_claim], source_notest=published_claim ) return ref def associate_wd_item(self, wd_item): """ Associate the data object with a Wikidata item. :param wd_item: Q-item that shall be assigned to the data object. :type wd_item: string """ if wd_item is not None: self.wd_item["wd-item"] = wd_item print("Associated WD item: ", wd_item) def add_label(self, language, text): """ Add a label in a specific language. :param language: code of language, e.g. "fi" :type language: string :param text: content of the label :type text: string """ base = self.wd_item["labels"] base.append({"language": language, "value": text}) def add_description(self, language, text): """ Add a description in a specific language. :param language: code of language, e.g. "fi" :type language: string :param text: content of the description :type text: string """ base = self.wd_item["descriptions"] base.append({"language": language, "value": text}) def construct_wd_item(self): """ Create the empty structure of the data object. This creates self.wd_item -- a dict container of all the data content of the item. """ self.wd_item = {} self.wd_item["upload"] = True self.wd_item["statements"] = [] self.wd_item["labels"] = [] self.wd_item["descriptions"] = [] self.wd_item["wd-item"] = None
class WikidataItem(object): def __init__(self, db_row_dict, repository, data_files, existing): self.repo = repository self.existing = existing self.wdstuff = WDS(self.repo) self.raw_data = db_row_dict self.props = data_files["properties"] self.construct_wd_item() self.problem_report = {} def make_q_item(self, qnumber): return self.wdstuff.QtoItemPage(qnumber) def make_pywikibot_item(self, value): val_item = None if isinstance(value, list) and len(value) == 1: value = value[0] if utils.string_is_q_item(value): val_item = self.make_q_item(value) elif value == "novalue": val_item = value elif isinstance(value, dict) and 'quantity_value' in value: number = value['quantity_value'] if 'unit' in value: unit = self.wdstuff.QtoItemPage(value["unit"]) else: unit = None val_item = pywikibot.WbQuantity(amount=number, unit=unit, site=self.repo) elif isinstance(value, dict) and 'date_value' in value: date_dict = value["date_value"] val_item = pywikibot.WbTime(year=date_dict.get("year"), month=date_dict.get("month"), day=date_dict.get("day")) elif value == "novalue": # raise NotImplementedError # implement Error print("Status: novalue will be added here") else: val_item = value return val_item def make_statement(self, value): if value in ['somevalue', 'novalue']: special = True else: special = False return self.wdstuff.Statement(value, special=special) def make_qualifier_applies_to(self, value): prop_item = self.props["applies_to_part"] target_item = self.wdstuff.QtoItemPage(value) return self.wdstuff.Qualifier(prop_item, target_item) def add_statement(self, prop_name, value, quals=None, ref=None): base = self.wd_item["statements"] prop = self.props[prop_name] if quals is None: quals = [] wd_claim = self.make_pywikibot_item(value) statement = self.make_statement(wd_claim) for qual in helpers.listify(quals): statement.addQualifier(qual) base.append({"prop": prop, "value": statement, "ref": ref}) def make_stated_in_ref(self, value, pub_date, ref_url=None, retrieved_date=None): item_prop = self.props["stated_in"] published_prop = self.props["publication_date"] pub_date = utils.date_to_dict(pub_date, "%Y-%m-%d") timestamp = self.make_pywikibot_item({"date_value": pub_date}) published_claim = self.wdstuff.make_simple_claim( published_prop, timestamp) source_item = self.wdstuff.QtoItemPage(value) source_claim = self.wdstuff.make_simple_claim(item_prop, source_item) if ref_url and retrieved_date: ref_url_prop = self.props["reference_url"] retrieved_date_prop = self.props["retrieved"] retrieved_date = utils.date_to_dict(retrieved_date, "%Y-%m-%d") retrieved_date = self.make_pywikibot_item( {"date_value": retrieved_date}) ref_url_claim = self.wdstuff.make_simple_claim( ref_url_prop, ref_url) retrieved_on_claim = self.wdstuff.make_simple_claim( retrieved_date_prop, retrieved_date) ref = self.wdstuff.Reference( source_test=[source_claim, ref_url_claim], source_notest=[published_claim, retrieved_on_claim]) else: ref = self.wdstuff.Reference(source_test=[source_claim], source_notest=published_claim) return ref def associate_wd_item(self, wd_item): if wd_item is not None: self.wd_item["wd-item"] = wd_item def add_label(self, language, text): base = self.wd_item["labels"] base.append({"language": language, "value": text}) def add_description(self, language, text): base = self.wd_item["descriptions"] base.append({"language": language, "value": text}) def construct_wd_item(self): self.wd_item = {} self.wd_item["upload"] = True self.wd_item["statements"] = [] self.wd_item["labels"] = [] self.wd_item["descriptions"] = [] self.wd_item["wd-item"] = None
def test_qualifier_init(self): q_int = WD.Qualifier(123, 'foo') q_no_p = WD.Qualifier('123', 'foo') q_with_p = WD.Qualifier('P123', 'foo') self.assertEqual(q_int, q_no_p) self.assertEqual(q_with_p, q_no_p)