def parse(legislature=None, dir_=None, force_reparse=False):
"Update db from votes in (legislature's) dir."
if legislature and dir_ is None:
dir_ = PDF.LOCAL_DIR % {'legislature':legislature}
if not force_reparse:
refs_parsed = set(s.ref for s in db.Scrutin.query\
.filter_by(legislature=legislature))
fnames = sorted(os.listdir(dir_))
c = it.count()
errors = {}
for fname in fnames:
pdf = PDF.from_filename(fname)
if not force_reparse and pdf.ref in refs_parsed:
continue
try:
populate_scrutin(pdf)
logger.info('Lecture %s OK' % pdf)
except Exception, e:
logger.info('Lecture %s - %s' % (pdf, e))
errors[pdf.filename] = e
continue
else:
try:
Session.commit()
except SQLAlchemyError, e:
logger.warning('Rolling back %s - %s' % (pdf, e))
Session.rollback()
else:
def add_canton_in_mandats_CF():
# canton manque ds mandats récents au CF
for m in db.Mandat.query.filter_by(canton=None):
p = m.personne
c = p.mandats[-1].canton
if c:
for m in p.mandats:
if not m.canton:
m.canton = c
logger.debug('%s canton -> %s' % (p, c))
Session.commit()
def import_from_csv(fname=os.path.join(DATA_DIR, 'Ratsmitglieder.csv')):
for pers, rows in read_csv(fname):
p = db.Personne.get_or_create(**pers._asdict())
for m in sorted((r.mandat for r in rows), key=lambda m: m.date_fin):
db.Parti.get_or_create(m.parti_f)
db.GroupeParlementaire.get_or_create(m.groupe_f)
db.Mandat.get_or_create(
personne_nom=p.nom,
personne_prenom=p.prenom,
personne_date_naissance=p.date_naissance,
date_debut=m.date_debut,
conseil=m.conseil,
canton=m.canton or None,
parti_ref=m.parti_f,
groupe_ref=m.groupe_f,
date_fin=m.date_fin)
Session.commit()
def parse(refs=None, force=False, limit=None):
if refs is None:
q = Session.query(ObjetParlementaire)
#.options(defer('description', 'beschreibung'))
if not force:
q = q.filter(ObjetParlementaire.etat==None)
if limit:
q = q.limit(limit)
refs = [o.ref for o in q]
dicts = map_async(parse_objet, refs)
for ref, vals in zip(refs, dicts):
o = ObjetParlementaire.get_or_create(ref)
for k,v in vals.iteritems():
setattr(o, k, v)
Session.commit()
logger.info('Updated %i `ObjetParlementaire`s' % len(refs))
.filter_by(legislature=legislature))
fnames = sorted(os.listdir(dir_))
c = it.count()
errors = {}
for fname in fnames:
pdf = PDF.from_filename(fname)
if not force_reparse and pdf.ref in refs_parsed:
continue
try:
populate_scrutin(pdf)
logger.info('Lecture %s OK' % pdf)
except Exception, e:
logger.info('Lecture %s - %s' % (pdf, e))
errors[pdf.filename] = e
continue
else:
try:
Session.commit()
except SQLAlchemyError, e:
logger.warning('Rolling back %s - %s' % (pdf, e))
Session.rollback()
else:
c.next()
logger.info(u'Importé %i scrutins (%i échec(s)) depuis %s' %\
(c.next(), len(errors), dir_))
if errors:
logger.debug(u'Échecs scrutins: %s' % errors)
Session.commit()
return errors