def write_list_of_stenograms_summary_pages(): logger_html.info("Generating html summary page of all stenograms.") all_stenograms_template = templates.get_template("stenograms_template.html") sessionscur = db.cursor() sessionscur.execute( """SELECT description, stenogram_date FROM vote_sessions ORDER BY stenogram_date, session_number""" ) date_stenogr = groupby_list(sessionscur, operator.itemgetter(1)) year_date_stenogr = groupby_list(date_stenogr, lambda d_s: d_s[0].year) years = zip(*year_date_stenogr)[0] for y, y_date_stenogr in year_date_stenogr: month_date_stenogr = [("all", y_date_stenogr)] + groupby_list(y_date_stenogr, lambda d_s: d_s[0].month) months = zip(*month_date_stenogr)[0] for m, m_date_stenogr in month_date_stenogr: with open("generated_html/stenograms%s%s.html" % (y, m), "w") as html_file: html_file.write( all_stenograms_template.render( years=years, months=months, current_y=y, stenogram_mgroup=m_date_stenogr ) ) sitemap.add("stenograms%s%s.html" % (y, m), 0.8) # Copy the most recent one os.system("cp generated_html/stenograms%s%s.html generated_html/stenograms.html" % (y, m)) sitemap.add("stenograms.html", 0.8)
def load_votes_regs_data(): global data_loaded if data_loaded: return data_loaded = True logger_html.info("Fetching most of the db in memory.") # All MPs mpscur = db.cursor() mpscur.execute( """SELECT mp_name, orig_party_name, (SELECT LAST(with_party ORDER BY mp_reg.stenogram_date) FROM mp_reg WHERE mp_reg.mp_name = mps.mp_name), original_url FROM mps ORDER BY orig_party_name, mp_name""" ) global mps, parties mps = mpscur.fetchall() parties = groupby_list(mps, operator.itemgetter(1)) # All sessions sescur = db.cursor() sescur.execute( """SELECT stenogram_date, session_number FROM vote_sessions ORDER BY stenogram_date, session_number""" ) global sessions, session_dates sessions = sescur.fetchall() session_dates = groupby_list(sessions, operator.itemgetter(0)) # All dates - includes dates on which no voting was done datecur = db.cursor() datecur.execute( """SELECT stenogram_date FROM stenograms ORDER BY stenogram_date""" ) global all_dates all_dates = [d[0] for d in datecur] def aggregate_sessions_in_dates(array): new_shape = list(array.shape) new_shape[1] = len(session_dates) new_array = np.zeros(new_shape, dtype=np.int32) start = 0 for date_i, date in enumerate(session_dates): end = start + len(date[1]) new_array[:, date_i, :] = np.sum(array[:, start:end, :], 1) start = end return new_array def aggregate_names_in_parties(array): new_shape = list(array.shape) new_shape[0] = len(parties) new_array = np.zeros(new_shape, dtype=np.int32) start = 0 for party_i, party in enumerate(parties): end = start + len(party[1]) new_array[party_i, :, :] = np.sum(array[start:end, :, :], 0) start = end return new_array # All regs. datacur = db.cursor() global mps_dates_reg mps_dates_reg = np.zeros((len(mps), len(all_dates), 3), dtype=np.int32) """ A 3D array with the same structure as described for `mps_sessions_vote`.""" reg_dict = {"present": 0, "absent": 1, "manually_registered": 2} for (mp_i, mp) in enumerate(mps): datacur.execute( """SELECT reg, stenogram_date FROM mp_reg WHERE mp_name = %s ORDER BY stenogram_date""", (mp[0],), ) f = datacur.fetchone() for r in datacur: date_i = index(all_dates, r[1]) # While the index is monotonic it is not always +=1 mps_dates_reg[mp_i, date_i, reg_dict[r[0]]] = 1 global mps_all_reg mps_all_reg = np.sum(mps_dates_reg, 1) # All votes global mps_sessions_vote mps_sessions_vote = np.zeros((len(mps), len(sessions), 4), dtype=np.int32) """ A 3D array sessions / with the index /___yes_no_abst_absent following | 0 0 0 1 structure: names | 1 0 0 0 index | ... Contains votes. If the MP was not even registered for the session it contains only zeros.""" vote_dict = {"yes": 0, "no": 1, "abstain": 2, "absent": 3} for (mp_i, mp) in enumerate(mps): datacur.execute( """SELECT vote, stenogram_date, session_number FROM mp_votes WHERE mp_name = %s ORDER BY stenogram_date, session_number""", (mp[0],), ) for v in datacur.fetchall(): ses_i = index(sessions, v[1:]) # While the index is monotonic it is not always +=1 mps_sessions_vote[mp_i, ses_i, vote_dict[v[0]]] = 1 global mps_dates_vote, parties_sessions_vote, parties_dates_vote, mps_all_vote, all_sessions_vote mps_dates_vote = aggregate_sessions_in_dates(mps_sessions_vote) parties_sessions_vote = aggregate_names_in_parties(mps_sessions_vote) parties_dates_vote = aggregate_sessions_in_dates(parties_sessions_vote) mps_all_vote = np.sum(mps_dates_vote, 1) all_sessions_vote = np.sum(parties_sessions_vote, 0) """3D and 2D arrays with aggregated data.""" global mps_sessions_with_against_party mps_sessions_with_against_party = np.zeros((len(mps), len(sessions), 2), dtype=np.int32) """ A 3D array with the same structure as above. The data is vote with/against party.""" offset = 0 for p_i, (party, p_mps) in enumerate(parties): end = offset + len(p_mps) for mp_i, ses_i in itertools.product(range(offset, end), range(len(sessions))): if not any(mps_sessions_vote[mp_i, ses_i, :2]): continue if parties_sessions_vote[p_i, ses_i, 0] == parties_sessions_vote[p_i, ses_i, 1]: mps_sessions_with_against_party[mp_i, ses_i, 0] = 1 continue mps_sessions_with_against_party[mp_i, ses_i, 0] = mps_sessions_vote[ mp_i, ses_i, np.argmax(parties_sessions_vote[p_i, ses_i, :2]) ] mps_sessions_with_against_party[mp_i, ses_i, 1] = 1 - mps_sessions_with_against_party[mp_i, ses_i, 0] offset = end global mps_dates_with_against_party, mps_all_with_against_party mps_dates_with_against_party = aggregate_sessions_in_dates(mps_sessions_with_against_party) mps_all_with_against_party = np.sum(mps_dates_with_against_party, 1) """3D and 2D arrays with aggregated data.""" global mps_sessions_with_against_all mps_sessions_with_against_all = np.zeros((len(mps), len(sessions), 2), dtype=np.int32) """ A 3D array with the same structure as above. The data is vote with/against all.""" for mp_i, ses_i in itertools.product(range(len(mps)), range(len(sessions))): if not any(mps_sessions_vote[mp_i, ses_i, :2]): continue if all_sessions_vote[ses_i, 0] == all_sessions_vote[ses_i, 1]: mps_sessions_with_against_all[mp_i, ses_i, 0] = 1 continue mps_sessions_with_against_all[mp_i, ses_i, 0] = mps_sessions_vote[ mp_i, ses_i, np.argmax(all_sessions_vote[ses_i, :2]) ] mps_sessions_with_against_all[mp_i, ses_i, 1] = 1 - mps_sessions_with_against_all[mp_i, ses_i, 0] global mps_dates_with_against_all, mps_all_with_against_all mps_dates_with_against_all = aggregate_sessions_in_dates(mps_sessions_with_against_all) mps_all_with_against_all = np.sum(mps_dates_with_against_all, 1) """3D and 2D arrays with aggregated data."""