def generate_person_long_names(engine):
log.info("Generating person fingerprints and slugs...")
from offenesparlament.transform.namematch import match_speaker
nkp = nk_persons()
Person = sl.get_table(engine, 'person')
for person in sl.find(engine, Person):
long_name = make_long_name(person)
try:
long_name = match_speaker(long_name)
except NKNoMatch:
pass
log.info(" -> %s" % long_name.strip())
slug = url_slug(long_name)
sl.upsert(engine, Person, {
'fingerprint': long_name,
'slug': slug,
'id': person['id']},
unique=['id'])
tries = 0
while True:
try:
nkp.ensure_value(long_name, data=person)
except ValueError, E:
log.warn('Exception: %s' % str(E))
tries = tries + 1
if tries > 5:
raise
else:
break
def extend_abstimmungen(engine):
log.info("Amending votes ...")
Abstimmung = sl.get_table(engine, 'abstimmung')
for data in sl.distinct(engine, Abstimmung, 'person'):
try:
fp = match_speaker(data['person'])
except NKInvalid, inv:
log.exception(ve)
continue
except NKNoMatch, nm:
fp = None
log.info("No match for: %s", data['person'])
def extend_abstimmungen(db, master):
log.info("Amending votes ...")
Abstimmung = db["abstimmung"]
prints = make_prints(db)
for data in Abstimmung.distinct("person"):
try:
fp = match_speaker(master, data["person"], prints)
if fp is not None:
Abstimmung.writerow(
{"person": data.get("person"), "fingerprint": fp}, unique_columns=["person"], bufferlen=100
)
except ValueError, ve:
log.exception(ve)
def load_profiles(engine):
doc = etree.parse(FEED_URL)
Person = sl.get_table(engine, 'person')
for profile in doc.findall('//PROFIL'):
name = profile.findtext('.//VORNAME')
name += ' ' + profile.findtext('.//NACHNAME')
partei = profile.findtext('.//PARTEI')
name += ' ' + PARTEI_MAPPING.get(partei, partei)
try:
fp = match_speaker(name)
sl.upsert(engine, Person,
{'awatch_url': profile.get('url'),
'fingerprint': fp},
unique=['fingerprint'])
except NKNoMatch:
pass
except NKInvalid:
pass
def make_person(beitrag, fp, engine):
from offenesparlament.transform.namematch import match_speaker
nkp = nk_persons()
try:
fp = match_speaker(fp)
person = {
'fingerprint': fp,
'vorname': beitrag['vorname'],
'nachname': beitrag['nachname'],
'ort': beitrag.get('ort'),
'ressort': beitrag.get('ressort'),
'land': beitrag.get('land'),
'fraktion': beitrag.get('fraktion')
}
sl.upsert(engine, sl.get_table(engine, 'person'), person,
unique=['fingerprint'])
nkp.ensure_value(fp, data=person)
except NKNoMatch:
pass
return fp
def identify_speaker(self, match):
return match_speaker(match)
def identify_speaker(self, match):
return match_speaker(self.master, match, self.prints)