def committees(
cache_name=None,
throttle=None,
no_commit=False,
):
from mptracker.scraper.committees import CommitteeScraper
patcher = TablePatcher(
models.MpCommittee,
models.db.session,
key_columns=['chamber_id', 'cdep_id'],
)
http_session = create_session(
cache_name=cache_name,
throttle=throttle and float(throttle),
)
scraper = CommitteeScraper(http_session)
with patcher.process(autoflush=1000, remove=True) as add:
for committee in scraper.fetch_committees():
add(committee.as_dict(['chamber_id', 'cdep_id', 'name']))
if no_commit:
logger.warn("Rolling back the transaction")
models.db.session.rollback()
else:
models.db.session.commit()
def committees(
cache_name=None,
throttle=None,
no_commit=False,
):
from mptracker.scraper.committees import CommitteeScraper
mandate_lookup = models.MandateLookup()
http_session = create_session(
cache_name=cache_name,
throttle=throttle and float(throttle),
)
scraper = CommitteeScraper(http_session)
committee_patcher = TablePatcher(
models.MpCommittee,
models.db.session,
key_columns=['chamber_id', 'cdep_id'],
)
membership_patcher = TablePatcher(
models.MpCommitteeMembership,
models.db.session,
key_columns=['mandate_id', 'mp_committee_id', 'interval'],
)
with committee_patcher.process(remove=True) as add_committee, \
membership_patcher.process(remove=True) as add_membership:
for committee in scraper.fetch_committees():
res = add_committee(
committee.as_dict(['chamber_id', 'cdep_id', 'name']),
)
if res.is_new:
models.db.session.flush()
mp_committee = res.row
for member in committee.current_members + committee.former_members:
if member.end_date and member.end_date < TERM_2012_START:
logger.warn(
"Membership end date is before the 2012 "
"term started, skipping: %r %r %r",
member.mp_name, committee.name, member.end_date,
)
continue
interval = DateRange(
member.start_date or TERM_2012_START,
member.end_date or date.max,
)
if interval.lower > interval.upper:
import pdb; pdb.set_trace()
mandate = mandate_lookup.find(
member.mp_name,
member.mp_ident.year,
member.mp_ident.number,
)
add_membership({
'role': member.role,
'interval': interval,
'mandate_id': mandate.id,
'mp_committee_id': mp_committee.id,
})
if no_commit:
logger.warn("Rolling back the transaction")
models.db.session.rollback()
else:
models.db.session.commit()