def iter_committees(proposal):
for activity_item in proposal.activity:
html = pq(activity_item.html)
for link in html.items('a'):
href = link.attr('href')
if href and href.startswith(COMMITTEE_URL_PREFIX):
args = url_args(href)
committee = (models.MpCommittee.query.filter_by(
chamber_id=int(args['cam'])).filter_by(
cdep_id=int(args['idc'])).first())
assert committee is not None
yield committee
def iter_committees(proposal):
for activity_item in proposal.activity:
html = pq(activity_item.html)
for link in html.items('a'):
href = link.attr('href')
if href and href.startswith(COMMITTEE_URL_PREFIX):
args = url_args(href)
committee = (
models.MpCommittee.query
.filter_by(chamber_id=int(args['cam']))
.filter_by(cdep_id=int(args['idc']))
.first()
)
assert committee is not None
yield committee
def iter_committees(proposal):
for activity_item in flask.json.loads(proposal.activity or '[]'):
html = pq(activity_item['html'])
for link in html.items('a'):
href = link.attr('href')
if href and href.startswith(COMMITTEE_URL_PREFIX):
args = url_args(href)
committee = (
models.MpCommittee.query
.filter_by(chamber_id=int(args['cam']))
.filter_by(cdep_id=int(args['idc']))
.first()
)
assert committee is not None, \
"No such committee: %r/%r" % (args['cam'], args['idc'])
yield committee
def controversy():
old_voting_sessions = set(
models.VotingSession.query
.filter(models.VotingSession.controversy_id != None)
.all()
)
controversy_map = {}
for line in get_gdrive_csv(CONTROVERSY_CSV_KEY):
cdeppk = url_args(line['link']).get('idv', type=int)
slug = line['slug']
if slug not in controversy_map:
controversy_map[slug] = {
'data': {
'slug': slug,
'title': line['title'],
},
'voting_session_rows': [],
}
voting_session = (
models.VotingSession.query
.filter_by(cdeppk=cdeppk)
.first()
)
controversy_map[slug]['voting_session_rows'].append(voting_session)
controversy_patcher = TablePatcher(
models.Controversy,
models.db.session,
key_columns=['slug'],
)
with controversy_patcher.process(remove=True) as add_controversy:
for controversy in controversy_map.values():
result = add_controversy(controversy['data'])
controversy['row'] = result.row
models.db.session.flush()
voting_session_patcher = TablePatcher(
models.VotingSession,
models.db.session,
key_columns=['id'],
)
new_voting_sessions = set()
with voting_session_patcher.process() as add_voting_session:
for controversy in controversy_map.values():
for voting_session in controversy['voting_session_rows']:
data = {
'id': voting_session.id,
'controversy_id': controversy['row'].id,
}
add_voting_session(data, create=False)
new_voting_sessions.add(voting_session)
for voting_session in old_voting_sessions - new_voting_sessions:
add_voting_session({
'id': voting_session.id,
'controversy_id': None,
})
models.db.session.commit()
def controversy():
import csv, requests, io, sqlalchemy as sa
url = flask.current_app.config['CONTROVERSY_CSV_URL']
resp = requests.get(url)
csv_file = csv.DictReader(io.StringIO(resp.text))
old_voting_sessions = set(
models.VotingSession.query
.filter(models.VotingSession.controversy_id != None)
.all()
)
controversy_map = {}
for line in csv_file:
cdeppk = url_args(line['link']).get('idv', type=int)
slug = line['slug']
if slug not in controversy_map:
controversy_map[slug] = {
'data': {
'slug': slug,
'title': line['title'],
},
'voting_session_rows': [],
}
voting_session = (
models.VotingSession.query
.filter_by(cdeppk=cdeppk)
.first()
)
controversy_map[slug]['voting_session_rows'].append(voting_session)
controversy_patcher = TablePatcher(
models.Controversy,
models.db.session,
key_columns=['slug'],
)
with controversy_patcher.process(remove=True) as add_controversy:
for controversy in controversy_map.values():
result = add_controversy(controversy['data'])
controversy['row'] = result.row
models.db.session.flush()
voting_session_patcher = TablePatcher(
models.VotingSession,
models.db.session,
key_columns=['id'],
)
new_voting_sessions = set()
with voting_session_patcher.process() as add_voting_session:
for controversy in controversy_map.values():
for voting_session in controversy['voting_session_rows']:
data = {
'id': voting_session.id,
'controversy_id': controversy['row'].id,
}
add_voting_session(data, create=False)
new_voting_sessions.add(voting_session)
for voting_session in old_voting_sessions - new_voting_sessions:
add_voting_session({
'id': voting_session.id,
'controversy_id': None,
})
models.db.session.commit()
def controversy():
import csv, requests, io, sqlalchemy as sa
url = flask.current_app.config['CONTROVERSY_CSV_URL']
resp = requests.get(url)
csv_file = csv.DictReader(io.StringIO(resp.text))
old_voting_sessions = set(
models.VotingSession.query.filter(
models.VotingSession.controversy_id != None).all())
controversy_map = {}
for line in csv_file:
cdeppk = url_args(line['link']).get('idv', type=int)
slug = line['slug']
if slug not in controversy_map:
controversy_map[slug] = {
'data': {
'slug': slug,
'title': line['title'],
},
'voting_session_rows': [],
}
voting_session = (models.VotingSession.query.filter_by(
cdeppk=cdeppk).first())
controversy_map[slug]['voting_session_rows'].append(voting_session)
controversy_patcher = TablePatcher(
models.Controversy,
models.db.session,
key_columns=['slug'],
)
with controversy_patcher.process(remove=True) as add_controversy:
for controversy in controversy_map.values():
result = add_controversy(controversy['data'])
controversy['row'] = result.row
models.db.session.flush()
voting_session_patcher = TablePatcher(
models.VotingSession,
models.db.session,
key_columns=['id'],
)
new_voting_sessions = set()
with voting_session_patcher.process() as add_voting_session:
for controversy in controversy_map.values():
for voting_session in controversy['voting_session_rows']:
data = {
'id': voting_session.id,
'controversy_id': controversy['row'].id,
}
add_voting_session(data, create=False)
new_voting_sessions.add(voting_session)
for voting_session in old_voting_sessions - new_voting_sessions:
add_voting_session({
'id': voting_session.id,
'controversy_id': None,
})
models.db.session.commit()
