def write_list_of_stenograms_summary_pages():
logger_html.info("Generating html summary page of all stenograms.")
all_stenograms_template = templates.get_template("stenograms_template.html")
sessionscur = db.cursor()
sessionscur.execute(
"""SELECT description, stenogram_date
FROM vote_sessions
ORDER BY stenogram_date, session_number"""
)
date_stenogr = groupby_list(sessionscur, operator.itemgetter(1))
year_date_stenogr = groupby_list(date_stenogr, lambda d_s: d_s[0].year)
years = zip(*year_date_stenogr)[0]
for y, y_date_stenogr in year_date_stenogr:
month_date_stenogr = [("all", y_date_stenogr)] + groupby_list(y_date_stenogr, lambda d_s: d_s[0].month)
months = zip(*month_date_stenogr)[0]
for m, m_date_stenogr in month_date_stenogr:
with open("generated_html/stenograms%s%s.html" % (y, m), "w") as html_file:
html_file.write(
all_stenograms_template.render(
years=years, months=months, current_y=y, stenogram_mgroup=m_date_stenogr
)
)
sitemap.add("stenograms%s%s.html" % (y, m), 0.8)
# Copy the most recent one
os.system("cp generated_html/stenograms%s%s.html generated_html/stenograms.html" % (y, m))
sitemap.add("stenograms.html", 0.8)
def load_votes_regs_data():
global data_loaded
if data_loaded:
return
data_loaded = True
logger_html.info("Fetching most of the db in memory.")
# All MPs
mpscur = db.cursor()
mpscur.execute(
"""SELECT mp_name,
orig_party_name,
(SELECT LAST(with_party ORDER BY mp_reg.stenogram_date) FROM mp_reg WHERE mp_reg.mp_name = mps.mp_name),
original_url
FROM mps
ORDER BY orig_party_name, mp_name"""
)
global mps, parties
mps = mpscur.fetchall()
parties = groupby_list(mps, operator.itemgetter(1))
# All sessions
sescur = db.cursor()
sescur.execute(
"""SELECT stenogram_date, session_number
FROM vote_sessions
ORDER BY stenogram_date, session_number"""
)
global sessions, session_dates
sessions = sescur.fetchall()
session_dates = groupby_list(sessions, operator.itemgetter(0))
# All dates - includes dates on which no voting was done
datecur = db.cursor()
datecur.execute(
"""SELECT stenogram_date
FROM stenograms
ORDER BY stenogram_date"""
)
global all_dates
all_dates = [d[0] for d in datecur]
def aggregate_sessions_in_dates(array):
new_shape = list(array.shape)
new_shape[1] = len(session_dates)
new_array = np.zeros(new_shape, dtype=np.int32)
start = 0
for date_i, date in enumerate(session_dates):
end = start + len(date[1])
new_array[:, date_i, :] = np.sum(array[:, start:end, :], 1)
start = end
return new_array
def aggregate_names_in_parties(array):
new_shape = list(array.shape)
new_shape[0] = len(parties)
new_array = np.zeros(new_shape, dtype=np.int32)
start = 0
for party_i, party in enumerate(parties):
end = start + len(party[1])
new_array[party_i, :, :] = np.sum(array[start:end, :, :], 0)
start = end
return new_array
# All regs.
datacur = db.cursor()
global mps_dates_reg
mps_dates_reg = np.zeros((len(mps), len(all_dates), 3), dtype=np.int32)
""" A 3D array with the same structure as described for `mps_sessions_vote`."""
reg_dict = {"present": 0, "absent": 1, "manually_registered": 2}
for (mp_i, mp) in enumerate(mps):
datacur.execute(
"""SELECT reg, stenogram_date FROM mp_reg
WHERE mp_name = %s
ORDER BY stenogram_date""",
(mp[0],),
)
f = datacur.fetchone()
for r in datacur:
date_i = index(all_dates, r[1]) # While the index is monotonic it is not always +=1
mps_dates_reg[mp_i, date_i, reg_dict[r[0]]] = 1
global mps_all_reg
mps_all_reg = np.sum(mps_dates_reg, 1)
# All votes
global mps_sessions_vote
mps_sessions_vote = np.zeros((len(mps), len(sessions), 4), dtype=np.int32)
""" A 3D array sessions /
with the index /___yes_no_abst_absent
following | 0 0 0 1
structure: names | 1 0 0 0
index | ...
Contains votes. If the MP was not even registered for the session it contains only zeros."""
vote_dict = {"yes": 0, "no": 1, "abstain": 2, "absent": 3}
for (mp_i, mp) in enumerate(mps):
datacur.execute(
"""SELECT vote, stenogram_date, session_number FROM mp_votes
WHERE mp_name = %s
ORDER BY stenogram_date, session_number""",
(mp[0],),
)
for v in datacur.fetchall():
ses_i = index(sessions, v[1:]) # While the index is monotonic it is not always +=1
mps_sessions_vote[mp_i, ses_i, vote_dict[v[0]]] = 1
global mps_dates_vote, parties_sessions_vote, parties_dates_vote, mps_all_vote, all_sessions_vote
mps_dates_vote = aggregate_sessions_in_dates(mps_sessions_vote)
parties_sessions_vote = aggregate_names_in_parties(mps_sessions_vote)
parties_dates_vote = aggregate_sessions_in_dates(parties_sessions_vote)
mps_all_vote = np.sum(mps_dates_vote, 1)
all_sessions_vote = np.sum(parties_sessions_vote, 0)
"""3D and 2D arrays with aggregated data."""
global mps_sessions_with_against_party
mps_sessions_with_against_party = np.zeros((len(mps), len(sessions), 2), dtype=np.int32)
""" A 3D array with the same structure as above. The data is vote with/against party."""
offset = 0
for p_i, (party, p_mps) in enumerate(parties):
end = offset + len(p_mps)
for mp_i, ses_i in itertools.product(range(offset, end), range(len(sessions))):
if not any(mps_sessions_vote[mp_i, ses_i, :2]):
continue
if parties_sessions_vote[p_i, ses_i, 0] == parties_sessions_vote[p_i, ses_i, 1]:
mps_sessions_with_against_party[mp_i, ses_i, 0] = 1
continue
mps_sessions_with_against_party[mp_i, ses_i, 0] = mps_sessions_vote[
mp_i, ses_i, np.argmax(parties_sessions_vote[p_i, ses_i, :2])
]
mps_sessions_with_against_party[mp_i, ses_i, 1] = 1 - mps_sessions_with_against_party[mp_i, ses_i, 0]
offset = end
global mps_dates_with_against_party, mps_all_with_against_party
mps_dates_with_against_party = aggregate_sessions_in_dates(mps_sessions_with_against_party)
mps_all_with_against_party = np.sum(mps_dates_with_against_party, 1)
"""3D and 2D arrays with aggregated data."""
global mps_sessions_with_against_all
mps_sessions_with_against_all = np.zeros((len(mps), len(sessions), 2), dtype=np.int32)
""" A 3D array with the same structure as above. The data is vote with/against all."""
for mp_i, ses_i in itertools.product(range(len(mps)), range(len(sessions))):
if not any(mps_sessions_vote[mp_i, ses_i, :2]):
continue
if all_sessions_vote[ses_i, 0] == all_sessions_vote[ses_i, 1]:
mps_sessions_with_against_all[mp_i, ses_i, 0] = 1
continue
mps_sessions_with_against_all[mp_i, ses_i, 0] = mps_sessions_vote[
mp_i, ses_i, np.argmax(all_sessions_vote[ses_i, :2])
]
mps_sessions_with_against_all[mp_i, ses_i, 1] = 1 - mps_sessions_with_against_all[mp_i, ses_i, 0]
global mps_dates_with_against_all, mps_all_with_against_all
mps_dates_with_against_all = aggregate_sessions_in_dates(mps_sessions_with_against_all)
mps_all_with_against_all = np.sum(mps_dates_with_against_all, 1)
"""3D and 2D arrays with aggregated data."""
