Exemple #1
0
def parse(legislature=None, dir_=None, force_reparse=False):
    "Update db from votes in (legislature's) dir."

    if legislature and dir_ is None:
        dir_ = PDF.LOCAL_DIR % {'legislature':legislature}

    if not force_reparse:
        refs_parsed = set(s.ref for s in db.Scrutin.query\
                                       .filter_by(legislature=legislature))
    fnames = sorted(os.listdir(dir_))
    c = it.count()
    errors = {}
    for fname in fnames:
        pdf = PDF.from_filename(fname)
        if not force_reparse and pdf.ref in refs_parsed:
            continue
        try:
            populate_scrutin(pdf)
            logger.info('Lecture %s OK' % pdf)
        except Exception, e:
            logger.info('Lecture %s - %s' % (pdf, e))
            errors[pdf.filename] = e
            continue
        else:
            try:
                Session.commit()
            except SQLAlchemyError, e:
                logger.warning('Rolling back %s - %s' % (pdf, e))
                Session.rollback()
            else:
Exemple #2
0
def add_canton_in_mandats_CF():
    # canton manque ds mandats récents au CF
    for m in db.Mandat.query.filter_by(canton=None):
        p = m.personne
        c = p.mandats[-1].canton
        if c:
            for m in p.mandats:
                if not m.canton:
                    m.canton = c
            logger.debug('%s canton -> %s' % (p, c))
    Session.commit()
Exemple #3
0
def import_from_csv(fname=os.path.join(DATA_DIR, 'Ratsmitglieder.csv')):
    for pers, rows in read_csv(fname):

        p = db.Personne.get_or_create(**pers._asdict())
        for m in sorted((r.mandat for r in rows), key=lambda m: m.date_fin):
            db.Parti.get_or_create(m.parti_f)
            db.GroupeParlementaire.get_or_create(m.groupe_f)
            db.Mandat.get_or_create(
                personne_nom=p.nom,
                personne_prenom=p.prenom,
                personne_date_naissance=p.date_naissance,
                date_debut=m.date_debut,
                conseil=m.conseil,
                canton=m.canton or None,
                parti_ref=m.parti_f,
                groupe_ref=m.groupe_f,
                date_fin=m.date_fin)
        Session.commit()
Exemple #4
0
def parse(refs=None, force=False, limit=None):

    if refs is None:
        q = Session.query(ObjetParlementaire)
            #.options(defer('description', 'beschreibung'))
        if not force:
            q = q.filter(ObjetParlementaire.etat==None)
        if limit:
            q = q.limit(limit)
        refs = [o.ref for o in q]

    dicts = map_async(parse_objet, refs)
    for ref, vals in zip(refs, dicts):
        o = ObjetParlementaire.get_or_create(ref)
        for k,v in vals.iteritems():
            setattr(o, k, v)
        Session.commit()
    logger.info('Updated %i `ObjetParlementaire`s' % len(refs))
Exemple #5
0
                                       .filter_by(legislature=legislature))
    fnames = sorted(os.listdir(dir_))
    c = it.count()
    errors = {}
    for fname in fnames:
        pdf = PDF.from_filename(fname)
        if not force_reparse and pdf.ref in refs_parsed:
            continue
        try:
            populate_scrutin(pdf)
            logger.info('Lecture %s OK' % pdf)
        except Exception, e:
            logger.info('Lecture %s - %s' % (pdf, e))
            errors[pdf.filename] = e
            continue
        else:
            try:
                Session.commit()
            except SQLAlchemyError, e:
                logger.warning('Rolling back %s - %s' % (pdf, e))
                Session.rollback()
            else:
                c.next()

    logger.info(u'Importé %i scrutins (%i échec(s)) depuis %s' %\
                (c.next(), len(errors), dir_))
    if errors:
        logger.debug(u'Échecs scrutins: %s' % errors)
    Session.commit()
    return errors