def parse(legislature=None, dir_=None, force_reparse=False): "Update db from votes in (legislature's) dir." if legislature and dir_ is None: dir_ = PDF.LOCAL_DIR % {'legislature':legislature} if not force_reparse: refs_parsed = set(s.ref for s in db.Scrutin.query\ .filter_by(legislature=legislature)) fnames = sorted(os.listdir(dir_)) c = it.count() errors = {} for fname in fnames: pdf = PDF.from_filename(fname) if not force_reparse and pdf.ref in refs_parsed: continue try: populate_scrutin(pdf) logger.info('Lecture %s OK' % pdf) except Exception, e: logger.info('Lecture %s - %s' % (pdf, e)) errors[pdf.filename] = e continue else: try: Session.commit() except SQLAlchemyError, e: logger.warning('Rolling back %s - %s' % (pdf, e)) Session.rollback() else:
def add_canton_in_mandats_CF(): # canton manque ds mandats récents au CF for m in db.Mandat.query.filter_by(canton=None): p = m.personne c = p.mandats[-1].canton if c: for m in p.mandats: if not m.canton: m.canton = c logger.debug('%s canton -> %s' % (p, c)) Session.commit()
def import_from_csv(fname=os.path.join(DATA_DIR, 'Ratsmitglieder.csv')): for pers, rows in read_csv(fname): p = db.Personne.get_or_create(**pers._asdict()) for m in sorted((r.mandat for r in rows), key=lambda m: m.date_fin): db.Parti.get_or_create(m.parti_f) db.GroupeParlementaire.get_or_create(m.groupe_f) db.Mandat.get_or_create( personne_nom=p.nom, personne_prenom=p.prenom, personne_date_naissance=p.date_naissance, date_debut=m.date_debut, conseil=m.conseil, canton=m.canton or None, parti_ref=m.parti_f, groupe_ref=m.groupe_f, date_fin=m.date_fin) Session.commit()
def parse(refs=None, force=False, limit=None): if refs is None: q = Session.query(ObjetParlementaire) #.options(defer('description', 'beschreibung')) if not force: q = q.filter(ObjetParlementaire.etat==None) if limit: q = q.limit(limit) refs = [o.ref for o in q] dicts = map_async(parse_objet, refs) for ref, vals in zip(refs, dicts): o = ObjetParlementaire.get_or_create(ref) for k,v in vals.iteritems(): setattr(o, k, v) Session.commit() logger.info('Updated %i `ObjetParlementaire`s' % len(refs))
.filter_by(legislature=legislature)) fnames = sorted(os.listdir(dir_)) c = it.count() errors = {} for fname in fnames: pdf = PDF.from_filename(fname) if not force_reparse and pdf.ref in refs_parsed: continue try: populate_scrutin(pdf) logger.info('Lecture %s OK' % pdf) except Exception, e: logger.info('Lecture %s - %s' % (pdf, e)) errors[pdf.filename] = e continue else: try: Session.commit() except SQLAlchemyError, e: logger.warning('Rolling back %s - %s' % (pdf, e)) Session.rollback() else: c.next() logger.info(u'Importé %i scrutins (%i échec(s)) depuis %s' %\ (c.next(), len(errors), dir_)) if errors: logger.debug(u'Échecs scrutins: %s' % errors) Session.commit() return errors