def generate_person_long_names(engine): log.info("Generating person fingerprints and slugs...") from offenesparlament.transform.namematch import match_speaker nkp = nk_persons() Person = sl.get_table(engine, 'person') for person in sl.find(engine, Person): long_name = make_long_name(person) try: long_name = match_speaker(long_name) except NKNoMatch: pass log.info(" -> %s" % long_name.strip()) slug = url_slug(long_name) sl.upsert(engine, Person, { 'fingerprint': long_name, 'slug': slug, 'id': person['id']}, unique=['id']) tries = 0 while True: try: nkp.ensure_value(long_name, data=person) except ValueError, E: log.warn('Exception: %s' % str(E)) tries = tries + 1 if tries > 5: raise else: break
def extend_abstimmungen(engine): log.info("Amending votes ...") Abstimmung = sl.get_table(engine, 'abstimmung') for data in sl.distinct(engine, Abstimmung, 'person'): try: fp = match_speaker(data['person']) except NKInvalid, inv: log.exception(ve) continue except NKNoMatch, nm: fp = None log.info("No match for: %s", data['person'])
def extend_abstimmungen(db, master): log.info("Amending votes ...") Abstimmung = db["abstimmung"] prints = make_prints(db) for data in Abstimmung.distinct("person"): try: fp = match_speaker(master, data["person"], prints) if fp is not None: Abstimmung.writerow( {"person": data.get("person"), "fingerprint": fp}, unique_columns=["person"], bufferlen=100 ) except ValueError, ve: log.exception(ve)
def load_profiles(engine): doc = etree.parse(FEED_URL) Person = sl.get_table(engine, 'person') for profile in doc.findall('//PROFIL'): name = profile.findtext('.//VORNAME') name += ' ' + profile.findtext('.//NACHNAME') partei = profile.findtext('.//PARTEI') name += ' ' + PARTEI_MAPPING.get(partei, partei) try: fp = match_speaker(name) sl.upsert(engine, Person, {'awatch_url': profile.get('url'), 'fingerprint': fp}, unique=['fingerprint']) except NKNoMatch: pass except NKInvalid: pass
def make_person(beitrag, fp, engine): from offenesparlament.transform.namematch import match_speaker nkp = nk_persons() try: fp = match_speaker(fp) person = { 'fingerprint': fp, 'vorname': beitrag['vorname'], 'nachname': beitrag['nachname'], 'ort': beitrag.get('ort'), 'ressort': beitrag.get('ressort'), 'land': beitrag.get('land'), 'fraktion': beitrag.get('fraktion') } sl.upsert(engine, sl.get_table(engine, 'person'), person, unique=['fingerprint']) nkp.ensure_value(fp, data=person) except NKNoMatch: pass return fp
def identify_speaker(self, match): return match_speaker(match)
def identify_speaker(self, match): return match_speaker(self.master, match, self.prints)