def proposals(
cache_name=None,
throttle=None,
autoanalyze=False,
):
from mptracker.scraper.proposals import ProposalScraper
from mptracker.proposals import ocr_proposal
from mptracker.policy import calculate_proposal
proposal_scraper = ProposalScraper(create_session(
cache_name=cache_name,
throttle=float(throttle) if throttle else None))
def cdep_id(mandate):
return (mandate.year, mandate.cdep_number)
by_cdep_id = {cdep_id(m): m
for m in models.Mandate.query
if m.year == 2012}
id_cdeppk_cdep = {}
id_cdeppk_senate = {}
for proposal in models.Proposal.query:
if proposal.cdeppk_cdep:
id_cdeppk_cdep[proposal.cdeppk_cdep] = proposal.id
if proposal.cdeppk_senate:
id_cdeppk_senate[proposal.cdeppk_senate] = proposal.id
chamber_by_slug = {c.slug: c for c in models.Chamber.query}
proposals = proposal_scraper.fetch_from_mp_pages(set(by_cdep_id.keys()))
all_activity = defaultdict(list)
for item in models.ProposalActivityItem.query:
all_activity[item.proposal_id].append(item)
proposal_patcher = TablePatcher(models.Proposal,
models.db.session,
key_columns=['id'])
activity_patcher = TablePatcher(models.ProposalActivityItem,
models.db.session,
key_columns=['id'])
sp_updates = sp_added = sp_removed = 0
changed = []
seen = []
with proposal_patcher.process(autoflush=1000, remove=True) as add_proposal:
with activity_patcher.process(autoflush=1000, remove=True) \
as add_activity:
for prop in proposals:
record = model_to_dict(prop, ['cdeppk_cdep', 'cdeppk_senate',
'decision_chamber', 'url', 'title', 'date', 'number_bpi',
'number_cdep', 'number_senate', 'proposal_type',
'pdf_url'])
slug = prop.decision_chamber
if slug:
record['decision_chamber'] = chamber_by_slug[slug]
idc = id_cdeppk_cdep.get(prop.cdeppk_cdep)
ids = id_cdeppk_senate.get(prop.cdeppk_senate)
if idc and ids and idc != ids:
logger.warn("Two different records for the same proposal: "
"(%s, %s). Removing the 2nd.", idc, ids)
models.db.session.delete(models.Proposal.query.get(ids))
ids = None
record['id'] = idc or ids or models.random_uuid()
result = add_proposal(record)
row = result.row
if result.is_changed:
changed.append(row)
seen.append(row)
new_people = set(by_cdep_id[ci] for ci in prop.sponsorships)
existing_sponsorships = {sp.mandate: sp
for sp in row.sponsorships}
to_remove = set(existing_sponsorships) - set(new_people)
to_add = set(new_people) - set(existing_sponsorships)
if to_remove:
logger.info("Removing sponsors %s: %r", row.id,
[cdep_id(m) for m in to_remove])
sp_removed += 1
for m in to_remove:
sp = existing_sponsorships[m]
models.db.session.delete(sp)
if to_add:
logger.info("Adding sponsors %s: %r", row.id,
[cdep_id(m) for m in to_add])
sp_added += 1
for m in to_add:
row.sponsorships.append(models.Sponsorship(mandate=m))
if to_remove or to_add:
sp_updates += 1
db_activity = all_activity[row.id]
db_activity.sort(key=lambda a: a.order)
act_fields = lambda r: (r.date, r.location)
if ([act_fields(r) for r in db_activity] !=
[act_fields(r) for r in prop.activity[:len(db_activity)]]):
logger.warn("History doesn't match for %s, "
"%d items will be removed",
row.id,len(db_activity))
db_activity = []
for n, ac in enumerate(prop.activity):
record = model_to_dict(ac, ['date', 'location', 'html'])
record['proposal_id'] = row.id
record['order'] = n
if n < len(db_activity):
item = db_activity[n]
record['id'] = item.id
assert item.date == record['date']
assert item.location == record['location']
assert item.order == record['order']
else:
record['id'] = models.random_uuid()
add_activity(record)
models.db.session.commit()
logger.info("Updated sponsorship for %d proposals (+%d, -%d)",
sp_updates, sp_added, sp_removed)
if autoanalyze:
logger.info("Scheduling analysis jobs for %d proposals", len(changed))
for proposal in changed:
if proposal.pdf_url:
ocr_proposal.delay(proposal.id, autoanalyze=True)
logger.info("Scheduling policy jobs for %d proposals", len(seen))
for proposal in seen:
if proposal.policy_domain_id is None:
calculate_proposal.delay(proposal.id)
def votes(
start=None,
days=1,
cache_name=None,
throttle=None,
no_commit=False,
autoanalyze=False,
):
from mptracker.scraper.votes import VoteScraper
if start is None:
start = models.db.session.execute(
'select date from voting_session '
'order by date desc limit 1').scalar() + ONE_DAY
else:
start = parse_date(start)
days = int(days)
http_session = create_session(cache_name=cache_name,
throttle=throttle and float(throttle))
vote_scraper = VoteScraper(http_session)
voting_session_patcher = TablePatcher(
models.VotingSession,
models.db.session,
key_columns=['cdeppk'],
)
vote_patcher = TablePatcher(
models.Vote,
models.db.session,
key_columns=['voting_session_id', 'mandate_id'],
)
proposal_ids = {p.cdeppk_cdep: p.id for p in models.Proposal.query}
mandate_lookup = models.MandateLookup()
new_voting_session_list = []
with voting_session_patcher.process() as add_voting_session:
with vote_patcher.process() as add_vote:
for delta in range(days):
the_date = start + ONE_DAY * delta
if the_date >= date.today():
# don't scrape today, maybe voting is not done yet!
break
logger.info("Scraping votes from %s", the_date)
for voting_session in vote_scraper.scrape_day(the_date):
record = model_to_dict(
voting_session,
['cdeppk', 'subject', 'subject_html'],
)
record['date'] = the_date
proposal_cdeppk = voting_session.proposal_cdeppk
record['proposal_id'] = (proposal_ids.get(proposal_cdeppk)
if proposal_cdeppk else None)
record['final'] = bool("vot final" in
record['subject'].lower())
vs = add_voting_session(record).row
if vs.id is None:
models.db.session.flush()
new_voting_session_list.append(vs.id)
for vote in voting_session.votes:
record = model_to_dict(vote, ['choice'])
record['voting_session_id'] = vs.id
mandate = mandate_lookup.find(
vote.mandate_name,
vote.mandate_year,
vote.mandate_number,
)
record['mandate_id'] = mandate.id
add_vote(record)
if no_commit:
logger.warn("Rolling back the transaction")
models.db.session.rollback()
else:
models.db.session.commit()
if autoanalyze:
from mptracker.votes import calculate_voting_session_loyalty
logger.info("Scheduling %d jobs", len(new_voting_session_list))
for voting_session_id in new_voting_session_list:
calculate_voting_session_loyalty.delay(voting_session_id)
def votes(
start=None,
n_days=1,
cache_name=None,
throttle=None,
):
from mptracker.scraper.votes import VoteScraper
if start is None:
start = models.db.session.execute(
'select date from voting_session '
'order by date desc limit 1').scalar() + ONE_DAY
else:
start = parse_date(start)
n_days = int(n_days)
http_session = create_session(cache_name=cache_name,
throttle=throttle and float(throttle))
vote_scraper = VoteScraper(http_session)
voting_session_patcher = TablePatcher(
models.VotingSession,
models.db.session,
key_columns=['cdeppk'],
)
vote_patcher = TablePatcher(
models.Vote,
models.db.session,
key_columns=['voting_session_id', 'mandate_id'],
)
proposal_ids = {p.cdeppk_cdep: p.id for p in models.Proposal.query}
mandate_lookup = models.MandateLookup()
with voting_session_patcher.process() as add_voting_session:
with vote_patcher.process() as add_vote:
for delta in range(n_days):
the_date = start + ONE_DAY * delta
if the_date >= date.today():
# don't scrape today, maybe voting is not done yet!
break
logger.info("Scraping votes from %s", the_date)
for voting_session in vote_scraper.scrape_day(the_date):
record = model_to_dict(
voting_session,
['cdeppk', 'subject', 'subject_html'],
)
record['date'] = the_date
proposal_cdeppk = voting_session.proposal_cdeppk
record['proposal_id'] = (proposal_ids.get(proposal_cdeppk)
if proposal_cdeppk else None)
vs = add_voting_session(record).row
if vs.id is None:
models.db.session.flush()
for vote in voting_session.votes:
record = model_to_dict(vote, ['choice'])
record['voting_session_id'] = vs.id
mandate = mandate_lookup.find(
vote.mandate_name,
vote.mandate_year,
vote.mandate_number,
)
record['mandate_id'] = mandate.id
add_vote(record)
models.db.session.commit()