def partipiskan():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
parties = s.query(Party).join(Member).join(ChamberAppointment) \
.filter(ChamberAppointment.start_date > dt.date(2010,10,5)).distinct().all()
for party in parties:
q = s.query(PartyVote, Poll).join(Poll).join(Party) \
.filter(Party.id==party.id)\
.order_by(Poll.date.asc())
print('{}'.format(party))
num_polls = 0
num_piska = 0
num_defectors = list()
for (pv, poll) in q:
counts = [pv.num_yes, pv.num_no, pv.num_abstain]
winner = max(counts)
total = sum(counts)
grand_total = total + pv.num_absent
num_polls += 1
if winner != total:
num_piska += 1
num_defectors.append(total - winner)
print('Piskan viner {}/{}'.format(num_piska, num_polls))
if len(num_defectors):
print('Max {}, medel {}'.format(
max(num_defectors),
sum(num_defectors) / len(num_defectors)))
def main():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
one_day = dt.timedelta(days=1)
f_trivial = codecs.open('chamber_consistency_trivial_overlaps.txt','w', encoding='utf-8')
f_bad = codecs.open('chamber_consistency_bad_overlaps.txt','w', encoding='utf-8')
f_empty_slots = codecs.open('chamber_consistency_empty_slots.txt','w', encoding='utf-8')
appointments = {app.id: app for app in s.query(ChamberAppointment)}
for chair in range(1,350):
out_intervals = compute_intervals(s, chair)
for i in out_intervals[1:]:
if len(i[2]) == 0:
f_empty_slots.write('Chair #{}: {} - {}\n'.format(chair, i[0],i[1]))
if len(i[2]) > 1:
overlap_msg = 'Chair #{}: {} - {}: {}\n'.format(chair,i[0],i[1], list(map(lambda x:appointments[x].title(),i[2])) )
if i[1]-i[0] >= one_day:
f_bad.write(overlap_msg)
else:
f_trivial.write(overlap_msg)
from demokratikollen.core.db_structure import *
from sqlalchemy import create_engine, func, distinct
from sqlalchemy.orm import sessionmaker, aliased
from sqlalchemy.sql.expression import literal
from itertools import combinations
from demokratikollen.core.utils import postgres as pg_utils
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
m1 = aliased(Member)
m2 = aliased(Member)
v1 = aliased(Vote)
v2 = aliased(Vote)
c = 0
for x in s.query(m1,m2,func.count(v1.poll_id))\
.filter(m1.id == v1.member_id)\
.filter(m2.id == v2.member_id)\
.filter(m1.id < m2.id)\
.filter(v1.poll_id == v2.poll_id)\
.filter(v1.vote_option == v2.vote_option)\
.filter(v1.vote_option != 'Frånvarande')\
.filter(v2.vote_option != 'Frånvarande')\
.group_by(m1.id,m2.id):
c += 1
print(x)
print(c)
def cosigning_timeseries():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
party_metadata = dict()
for party in s.query(Party):
party_metadata[party.id] = dict(abbr=party.abbr,
name=party.name,
ordering=party_ordering[party.abbr])
committee_metadata = dict()
for c in s.query(Committee):
committee_metadata[c.id] = dict(abbr=c.abbr, name=c.name)
committee_metadata[0] = dict(abbr="Alla", name="Alla utskott")
basequery = s.query(MemberProposal) \
.options(joinedload(MemberProposal.points).joinedload(ProposalPoint.committee_report).joinedload(CommitteeReport.committee)) \
.filter(MemberProposal.subtype != 'Enskild motion')
riksmoten = [
'{:04d}/{:02d}'.format(y, y - 2000 + 1) for y in range(2002, 2014)
]
output = dict()
num_missing_committee_report = 0
num_missing_committee_report_committee = 0
matrix = dict()
for (rm_idx, rm) in enumerate(riksmoten):
print("RM: {}".format(rm))
for doc in basequery.filter(MemberProposal.session == rm):
if not doc.signatories or len(doc.signatories) == 1:
continue
signing_parties = sorted(
list(
set(m.party.id for m in doc.signatories
if not m.party.abbr == "-")))
if len(signing_parties) <= 1:
continue
parties_key = repr(signing_parties)
for point in doc.points:
if not point.committee_report:
num_missing_committee_report += 1
continue
if not point.committee_report.committee:
num_missing_committee_report_committee += 1
continue
committee = point.committee_report.committee
# compute party matrix data
for (p1, p2) in permutations(signing_parties, 2):
if not p1 in matrix:
matrix[p1] = dict()
for committee_key in [0, committee.id]:
if not committee_key in matrix[p1]:
matrix[p1][committee_key] = dict()
if not p2 in matrix[p1][committee_key]:
matrix[p1][committee_key][p2] = dict(
values=[0 for rm in riksmoten],
abbr=party_metadata[p2]["abbr"],
name=party_metadata[p2]["name"],
id=p2)
matrix[p1][committee_key][p2]["values"][rm_idx] += 1
# compute timeseries data
for committee_key in [0, committee.id]:
if not committee_key in output:
output[committee_key] = dict()
if not parties_key in output[committee_key]:
output[committee_key][parties_key] = \
dict(
values = [0 for rm in riksmoten],
abbr = ' + '.join(party_metadata[p_id]["abbr"] for p_id in signing_parties),
name = ' + '.join(party_metadata[p_id]["name"] for p_id in signing_parties),
num_parties = len(signing_parties)
)
output[committee_key][parties_key]["values"][rm_idx] += 1
ds = MongoDBDatastore()
output_timeseries_top = dict(
t=riksmoten,
committees=[
dict(abbr=committee_metadata[c_id]["abbr"],
name=committee_metadata[c_id]["name"],
id=c_id,
series=[[
dict(abbr=p_dict["abbr"],
name=p_dict["name"],
num_parties=p_dict["num_parties"],
value=p_dict["values"][rm_idx])
for (p_key, p_dict) in series.items()
if p_dict["values"][rm_idx] > 0
] for (rm_idx, rm) in enumerate(riksmoten)])
for (c_id, series) in output.items()
])
ds.store_object(output_timeseries_top, "party_cosigning_timeseries")
mongodb = ds.get_mongodb_database()
mongo_collection = mongodb.party_cosigning_matrix
mongo_collection.ensure_index([("partyA", ASCENDING)], unique=True)
for (p1, matrix_p1) in matrix.items():
mongo_partyA = party_metadata[p1]["abbr"]
output_party_matrix_top = dict(
partyA=mongo_partyA,
t=riksmoten,
committees=[
dict(abbr=committee_metadata[c_id]["abbr"],
name=committee_metadata[c_id]["name"],
id=c_id,
parties=list(committee_dict.values()))
for (c_id, committee_dict) in matrix_p1.items()
])
mongo_collection.update(dict(partyA=mongo_partyA),
output_party_matrix_top,
upsert=True)
print(
"missing committee_report: {}, committee_reports missing committee's: {}"
.format(num_missing_committee_report,
num_missing_committee_report_committee))
def main():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
party_metadata = dict()
for party in s.query(Party):
party_metadata[party.id] = dict(
abbr=party.abbr,
name=party.name,
ordering=party_ordering[party.abbr]
)
committee_metadata = dict()
for c in s.query(Committee):
committee_metadata[c.id] = dict(
abbr=c.abbr,
name=c.name
)
result_metadata = dict()
result_metadata['bifall'] = dict(
name= 'Bifall',
ordering = 2
)
result_metadata['delvis bifall'] = dict(
name= 'Delvis bifall',
ordering = 1
)
result_metadata['avslag'] = dict(
name= 'Avslag',
ordering = 0
)
mdb = MongoDBDatastore()
mongodb = mdb.get_mongodb_database()
mongo_collection = mongodb.proposals_main
mongo_collection.ensure_index([("government", ASCENDING)], unique=True)
mongo_collection_party_detail = mongodb.proposals_party_detail
mongo_collection_party_detail.ensure_index([("government", ASCENDING),("party_id", ASCENDING)], unique=True)
mongo_collection_committee_detail = mongodb.proposals_committee_detail
mongo_collection_committee_detail.ensure_index([("government", ASCENDING),("committee_id", ASCENDING)], unique=True)
mongo_collection_ministries_detail = mongodb.proposals_ministries_detail
mongo_collection_ministries_detail.ensure_index([("government", ASCENDING)], unique=True)
mongo_collection_multiparties_detail = mongodb.proposals_multiparties_detail
mongo_collection_multiparties_detail.ensure_index([("government", ASCENDING)], unique=True)
mongo_collection_members_detail = mongodb.proposals_members_detail
mongo_collection_members_detail.ensure_index([("government", ASCENDING)], unique=True)
basequery = s.query(Proposal).options( \
joinedload(Proposal.points).joinedload(ProposalPoint.committee_report).joinedload(CommitteeReport.committee)) \
.order_by(Proposal.published.desc())
governments = [
("persson3",
dict(title="Person III", years="2002-2006", party="S"),
basequery.filter(MemberProposal.session.in_(['2002/03', '2003/04', '2004/05', '2005/06']))
),
("reinfeldt1",
dict(title="Reinfeldt I", years="2006-2010", party="M"),
basequery.filter(MemberProposal.session.in_(['2006/07', '2007/08', '2008/09', '2009/10']))
),
("reinfeldt2",
dict(title="Reinfeldt II", years="2010-2014", party="M"),
basequery.filter(MemberProposal.session.in_(['2010/11', '2011/12', '2012/13', '2013/14']))
)
]
num_missing_committee_report = 0
num_missing_committee_report_committee = 0
num_unknown_result = 0
for (name, government_metadata, query) in governments:
print("Government: {}".format(name))
parties_tree = Tree()
multiparties_tree = Tree()
ministries_tree = Tree()
members_tree = Tree()
committees_set = set()
results_set = set()
for doc in query:
if hasattr(doc, 'ministry_id'):
origin_tree = ministries_tree
origin_key = "Alla departement"
else:
if not doc.signatories or len(doc.signatories) == 0:
continue
signing_parties = sorted(list(set( m.party.id for m in doc.signatories)))
if doc.subtype == 'Enskild motion':
origin_tree = members_tree
origin_key = "Alla enskilda motioner"
elif len(signing_parties) == 1:
origin_tree = parties_tree
origin_key = signing_parties[0]
else: #multiparty signature
origin_tree = multiparties_tree
origin_key = "Alla Flerpartiförslag"
for point in doc.points:
if not point.committee_report:
print('Missing committee_report: {}, {}'.format(doc, point.number))
num_missing_committee_report += 1
continue
if not point.committee_report.committee:
print('Missing committee_report.committee: {}, {}'.format(doc, point.number))
num_missing_committee_report_committee += 1
continue
committee = point.committee_report.committee
result = point.decision
committee_key = committee.id
result_key = result.lower()
if not result_key in result_metadata.keys():
num_unknown_result += 1
print("Unknown result: {} {}".format(result_key, doc))
continue
if not result_key in origin_tree[origin_key][committee_key]:
origin_tree[origin_key][committee_key][result_key] = list()
origin_tree[origin_key][committee_key][result_key].append((doc.id, point.number))
committees_set.add(committee_key)
results_set.add(result_key)
def party_sort_key(p):
return party_metadata[p]['ordering']
def result_sort_key(r):
return result_metadata[r]['ordering']
parties = list(parties_tree.keys())
parties.sort(key=party_sort_key)
party_idc = {p:k for (k,p) in enumerate(parties)}
multiparties = list(multiparties_tree.keys())
multiparty_idc = {p:k for (k,p) in enumerate(multiparties)}
ministries = list(ministries_tree.keys())
ministry_idc = {p:k for (k,p) in enumerate(ministries)}
members = list(members_tree.keys())
member_idc = {p:k for (k,p) in enumerate(members)}
committees = list(committees_set)
committee_idc = {c:k for (k,c) in enumerate(committees)}
results = list(results_set)
results.sort(key=result_sort_key)
result_idc = {r:k for (k,r) in enumerate(results)}
nodes = list()
node_group_members = dict(
title="Enskilda\nledamöters\nmotioner",
items=[dict(detail="members", title=p) for p in members],
label = -1)
node_group_party = dict(
title="Partiernas\nmotioner",
items=[dict(detail="party", party_id=p, title=party_metadata[p]['name'],abbr=party_metadata[p]['abbr']) for p in parties],
label = -1)
node_group_multiparty = dict(
title="Flerparti-\nmotioner",
items=[dict(detail="multiparties", title=p) for p in multiparties],
label = -1)
node_group_government = dict(
title="Regeringens\npropositioner",
items=[dict(detail="government", title=p) for p in ministries],
label = -1)
node_group_committee = dict(
title="Utskotten",
items=[dict(detail="committee",committee_id=p, title=committee_metadata[p]['name'],abbr=committee_metadata[p]['abbr']) for p in committees],
label = 0)
node_group_results = dict(
title="Beslut",
items=[dict(detail="beslut", result_id=r, title=result_metadata[r]['name']) for r in results],
label = 0)
nodes = [
dict(x=0.0, title='1. Förslag kommer från\nriksdagens ledamöter\noch regeringen', items=[node_group_members, node_group_party, node_group_multiparty, node_group_government]),
dict(x=0.5, title='2. De bereds i ett\nav utskotten.', items=[node_group_committee]),
dict(x=1.0, title='3. Beslut i\nriksdagens kammare.', items=[node_group_results])
]
flows = list()
for member_key in members_tree:
for committee_key in members_tree[member_key]:
for (result_key, point_ids) in members_tree[member_key][committee_key].items():
member_addr = (0,0,member_idc[member_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[member_addr, committee_addr, result_addr], magnitude=len(point_ids)))
for party_key in parties_tree:
for committee_key in parties_tree[party_key]:
for (result_key, point_ids) in parties_tree[party_key][committee_key].items():
party_addr = (0,1,party_idc[party_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[party_addr, committee_addr, result_addr], magnitude=len(point_ids)))
for multiparty_key in multiparties_tree:
for committee_key in multiparties_tree[multiparty_key]:
for (result_key, point_ids) in multiparties_tree[multiparty_key][committee_key].items():
multiparty_addr = (0,2,multiparty_idc[multiparty_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[multiparty_addr, committee_addr, result_addr], magnitude=len(point_ids)))
for ministry_key in ministries_tree:
for committee_key in ministries_tree[ministry_key]:
for (result_key, point_ids) in ministries_tree[ministry_key][committee_key].items():
ministry_addr = (0,3,ministry_idc[ministry_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[ministry_addr, committee_addr, result_addr], magnitude=len(point_ids)))
def parse_title(s,dbs):
r = re.sub(r'med anledning av', 'm.a.a.', s)
r = re.sub(r'([0-9]{4}/[0-9]{2}):(\S{,7})', lambda m: '<a href="{}">{}</a>'.format(riksdagen.url_from_code(m.group(1),m.group(2),dbs),m.group(0)), r)
return r
def author_string(doc):
if len(doc.signatories)>2:
return '{} och {} till'.format(repr(doc.signatories[0]),len(doc.signatories)-1)
else:
return ' och '.join(map(repr,doc.signatories))
def shortest_number_string(l):
items = []
i = 0
on_streak = False
while i < len(l):
if not on_streak:
start = i
if len(l) == i+1 or l[i+1] != l[i]+1:
if start == i:
items.append(repr(l[start]))
else:
if i-start == 1:
items.append('{},{}'.format(repr(l[start]),repr(l[i])))
else:
items.append('{}-{}'.format(repr(l[start]),repr(l[i])))
on_streak = False
else:
on_streak = True
i += 1
return ', '.join(items)
def documents_from_points(decision_sets):
result = list()
prevdocid = 0
row = None
for (decision, ids) in decision_sets:
for (docid, number) in ids:
if not (docid == prevdocid):
doc = s.query(Document).get(docid)
parsed_title = parse_title(doc.title,s)
authors = author_string(doc)
title_html = '<span class="authors">{}</span><span class="title">{}</span>'.format(authors, parsed_title)
row = dict(
decision=decision,
title=title_html,
unique_code=doc.unique_code(),
url=riksdagen.url_from_dokid(doc.dok_id),
numbers='')
numbers = []
result.append(row)
numbers.append(number)
numbers.sort()
row['numbers'] = shortest_number_string(numbers)
prevdocid = docid
return result
def documents_from_prop(decision_sets):
result = list()
prevdocid = 0
row = None
for (decision, ids) in decision_sets:
for (docid, number) in ids:
if not (docid == prevdocid):
doc = s.query(Document).get(docid)
parsed_title = parse_title(doc.title,s)
authors = "?"
if not doc.ministry is None:
authors = doc.ministry.name
title_html = '<span class="authors">{}</span><span class="title">{}</span>'.format(authors, parsed_title)
row = dict(
decision=decision,
title=title_html,
unique_code=doc.unique_code(),
url=riksdagen.url_from_dokid(doc.dok_id),
numbers='')
numbers = []
result.append(row)
numbers.append(number)
numbers.sort()
row['numbers'] = shortest_number_string(numbers)
prevdocid = docid
return result
def committee_detail_for_tree(tree, doctype):
total_bifall = 0
total_delvis_bifall = 0
total_avslag = 0
committee_results = []
for c_id in committees:
if c_id not in tree:
continue
bifall_ids = tree[c_id].get('bifall',[])
delvis_bifall_ids = tree[c_id].get('delvis bifall',[])
avslag_ids = tree[c_id].get('avslag',[])
num_bifall = len(bifall_ids)
num_delvis_bifall = len(delvis_bifall_ids)
num_avslag = len(avslag_ids)
total_bifall += num_bifall
total_delvis_bifall += num_delvis_bifall
total_avslag += num_avslag
all_ids = (("Bifall", bifall_ids), ("Delvis bifall", delvis_bifall_ids), ("Avslag", avslag_ids))
doc_function = documents_from_points
if doctype == 'government':
all_ids = all_ids[::-1]
doc_function = documents_from_prop
committee_results.append({
'name': committee_metadata[c_id]['name'],
'abbr': committee_metadata[c_id]['abbr'],
'bifall': num_bifall,
'delvis_bifall': num_delvis_bifall,
'avslag': num_avslag,
'documents': doc_function(all_ids)
})
return dict(
committee_results = committee_results,
total_bifall = total_bifall,
total_delvis_bifall = total_delvis_bifall,
total_avslag = total_avslag
)
def origin_detail_for_tree(tree, doctype):
origin_results = []
bifall_ids = tree[c_id].get('bifall',[])
delvis_bifall_ids = tree[c_id].get('delvis bifall',[])
avslag_ids = tree[c_id].get('avslag',[])
num_bifall = len(bifall_ids)
num_delvis_bifall = len(delvis_bifall_ids)
num_avslag = len(avslag_ids)
all_ids = (("Bifall", bifall_ids), ("Delvis bifall", delvis_bifall_ids), ("Avslag", avslag_ids))
doc_function = documents_from_points
if doctype == 'government':
all_ids = all_ids[::-1]
doc_function = documents_from_prop
return {
'bifall': num_bifall,
'delvis_bifall': num_delvis_bifall,
'avslag': num_avslag,
'documents': doc_function(all_ids)
}
print('Generating party_detail')
party_detail = dict();
for p_id in parties:
committee_detail = committee_detail_for_tree(parties_tree[p_id], 'motion')
committee_detail['party'] = party_metadata[p_id]
party_detail[repr(p_id)] = committee_detail
print('Generating ministries_detail')
ministries_detail = committee_detail_for_tree(ministries_tree[ministries[0]], 'government')
print('Generating members_detail')
members_detail = committee_detail_for_tree(members_tree[members[0]],'motion')
print('Generating multiparties_detail')
multiparties_detail = committee_detail_for_tree(multiparties_tree[multiparties[0]],'motion')
print('Generating committee_detail')
committee_detail = dict();
for c_id in committees:
total_bifall = 0
total_delvis_bifall = 0
total_avslag = 0
origin_results = []
# government proposals first
ret = origin_detail_for_tree(ministries_tree[ministries[0]], 'government')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name='Regeringens propositioner',
abbr='R'
))
origin_results.append(ret)
# then members
ret = origin_detail_for_tree(members_tree[members[0]], 'motion')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name='Enskilda ledamöters motioner',
abbr='elm'
))
origin_results.append(ret)
# then multiparties
ret = origin_detail_for_tree(multiparties_tree[multiparties[0]], 'motion')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name='Samarbetsförslag',
abbr='sf'
))
origin_results.append(ret)
# finally parties
for p_id in parties:
if c_id not in parties_tree[p_id]:
continue
ret = origin_detail_for_tree(parties_tree[p_id], 'motion')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name=party_metadata[p_id]['name'],
abbr=party_metadata[p_id]['abbr']
))
origin_results.append(ret)
committee_detail[repr(c_id)] = dict(
committee = committee_metadata[c_id],
origin_results = origin_results,
total_bifall = total_bifall,
total_delvis_bifall = total_delvis_bifall,
total_avslag = total_avslag
)
output_top = dict(
government=name,
nodes = nodes,
flows = flows
)
output_top.update(government_metadata)
mongo_collection.update(dict(government=name), output_top, upsert=True)
ministries_detail.update(dict(government=name))
mongo_collection_ministries_detail.update(dict(government=name), ministries_detail, upsert=True)
members_detail.update(dict(government=name))
mongo_collection_members_detail.update(dict(government=name), members_detail, upsert=True)
multiparties_detail.update(dict(government=name))
mongo_collection_multiparties_detail.update(dict(government=name), multiparties_detail, upsert=True)
for (p_id, d) in party_detail.items():
d.update(dict(
government=name,party_id = p_id,
))
mongo_collection_party_detail.update(dict(government=name,party_id = p_id), d, upsert=True)
for (c_id, d) in committee_detail.items():
d.update(dict(
government=name,committee_id = c_id,
))
mongo_collection_committee_detail.update(dict(government=name,committee_id = c_id), d, upsert=True)
print("missing committee_report: {}\ncommittee_reports missing committee's: {}\nunknown result: {}".format(num_missing_committee_report,num_missing_committee_report_committee,num_unknown_result))
def main():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
# Enumerate all searchable objects and arrange in groups
members = list()
member_indices = dict()
for (idx, member) in enumerate(s.query(Member)):
members.append(
dict(title=member.first_name + " " + member.last_name + " (" +
member.party.abbr + ")",
img_url=member.image_url_sm,
url='/' + member.url_name))
member_indices[member.id] = idx
parties = list()
party_indices = dict()
for (idx, party) in enumerate(s.query(Party)):
parties.append(
dict(title=party.name,
img_url='/static/img/parties/' + party.abbr.lower() + '.png',
url='/' + party.abbr.lower()))
party_indices[party.id] = idx
output_groups = [
dict(title="Ledamöter", objects=members),
dict(title="Partier", objects=parties)
]
group_indices = dict(member=0, party=1)
# Enumerate all strings to match and couple them to objects (primary and secondary hits)
keywords = list()
for member in s.query(Member):
normalized_string = (member.first_name + " " +
member.last_name).lower()
keywords.append(
dict(string=normalized_string,
primary=(group_indices["member"], member_indices[member.id]),
secondaries=[(group_indices["party"],
party_indices[member.party.id])]))
for party in s.query(Party):
normalized_string = (party.name).lower()
keywords.append(
dict(string=normalized_string,
primary=(group_indices["party"], party_indices[party.id]),
secondaries=[(group_indices["member"], member_indices[m.id])
for m in party.members[0:2]]))
# Generate reverse table of all trigrams to match, such that we can ask:
# which keywords contain this trigram (and how many)
trigrams_dict = dict()
for (keyword_idx, keyword) in enumerate(keywords):
grams = Counter(
map(lambda t: ''.join(t),
zip(*[keyword["string"][k:] for k in range(3)])))
for (gram, count) in grams.items():
if not gram in trigrams_dict:
trigrams_dict[gram] = list()
trigrams_dict[gram].append((keyword_idx, count))
print('# unique trigrams: {}'.format(len(trigrams_dict)))
output_top = dict(groups=output_groups, keywords=keywords)
ds = MongoDBDatastore()
ds.store_object(output_top, "search")
def main():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
riksmoten = ['{:04d}/{:02d}'.format(y,y-2000+1) for y in range(2002,2014)]
parties = s.query(Party) \
.filter(Party.abbr != "-") \
.order_by(Party.id)
v1 = aliased(PartyVote)
v2 = aliased(PartyVote)
v3 = aliased(PartyVote)
mdb = MongoDBDatastore()
mongodb = mdb.get_mongodb_database()
mongo_collection = mongodb.party_covoting
mongo_collection.ensure_index([("partyA",ASCENDING),("partyB",ASCENDING)],unique=True)
for (partyA, partyB) in combinations(parties, 2):
conflicting_votes = s.query(v1, v2) \
.filter(v1.polled_point_id == v2.polled_point_id) \
.filter(v1.party_id == partyA.id) \
.filter(v2.party_id == partyB.id) \
.filter(or_(v1.vote_option == 'Ja', v1.vote_option == 'Nej')) \
.filter(or_(v2.vote_option == 'Ja', v2.vote_option == 'Nej')) \
.filter(v1.vote_option != v2.vote_option)
num_conflicting = conflicting_votes.count()
if num_conflicting < 1000:
continue
print('================================')
print('{} vs {}: {} conflicting votes'.format(partyA.abbr,partyB.abbr, num_conflicting))
output_parties = list()
for party in parties:
print('{}...'.format(party.abbr))
agree_query = s.query(v1, v2, v3, PolledPoint,CommitteeReport) \
.filter(CommitteeReport.id == PolledPoint.committee_report_id) \
.filter(PolledPoint.id == v1.polled_point_id) \
.filter(v1.polled_point_id == v2.polled_point_id) \
.filter(v1.party_id == partyA.id) \
.filter(v2.party_id == partyB.id) \
.filter(or_(v1.vote_option == 'Ja', v1.vote_option == 'Nej')) \
.filter(or_(v2.vote_option == 'Ja', v2.vote_option == 'Nej')) \
.filter(v1.vote_option != v2.vote_option) \
.filter(v3.polled_point_id == v2.polled_point_id) \
.filter(v3.party_id == party.id)
party_bias = list()
for (rm_idx, rm) in enumerate(riksmoten):
agree_query_interval = agree_query.filter(CommitteeReport.session == rm)
agreeA = agree_query_interval.filter(v3.vote_option == v1.vote_option).count()
agreeB = agree_query_interval.filter(v3.vote_option == v2.vote_option).count()
print(rm,agreeA,agreeB)
if agreeA + agreeB > 0:
party_bias.append( float(agreeB - agreeA)/float(agreeA + agreeB))
else:
party_bias.append(None)
if all(x is None for x in party_bias):
continue
output_parties.append(dict(
party_id=party.id,
party_abbr=party.abbr,
party_name=party.name,
values=party_bias))
print('Dumping to MongoDB.')
output_parties_reverse = deepcopy(output_parties)
for item in output_parties_reverse:
item['values'] = [x if x is None else -x for x in item['values']]
output_top = dict(partyA = partyA.id, partyB = partyB.id, series = output_parties,labels=riksmoten)
output_top_reverse = dict(partyA = partyB.id, partyB = partyA.id, series = output_parties_reverse, labels=riksmoten)
mongo_collection.update(dict(partyA = partyA.id, partyB = partyB.id), output_top, upsert=True)
mongo_collection.update(dict(partyA = partyB.id, partyB = partyA.id), output_top_reverse, upsert=True)
def poll_freq():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
weekday = {k: 0 for k in range(1,8)}
week = {k: 0 for k in range(1,54)}
month = {k: 0 for k in range(1,13)}
day = {k: 0 for k in range(1,32)}
exact = dict()
for poll in s.query(Poll).all():
weekday[poll.date.isoweekday()] += 1
week[poll.date.isocalendar()[1]] += 1
month[poll.date.month] += 1
day[poll.date.day] += 1
timestamp = dt.datetime.combine(poll.date,dt.time(0)).timestamp()
if timestamp in exact:
exact[timestamp] += 1
else:
exact[timestamp] = 1
datastore = mongo_utils.MongoDBDatastore()
datastore.store_string(json.dumps(weekday),'poll_frequency_weekday')
datastore.store_string(json.dumps(month),'poll_frequency_month')
datastore.store_string(json.dumps(month),'poll_frequency_week')
datastore.store_string(json.dumps(day),'poll_frequency_day')
datastore.store_string(json.dumps(exact),'poll_frequency_exact')
fig = plt.figure(0, figsize=(10,10))
ax = fig.add_subplot(111)
xs = np.arange(1,8)-0.4
ax.bar(xs, [weekday[i] for i in range(1,8)])
ax.set_xticks(np.arange(1,8))
fig.savefig('pollfreq_weekday.png')
plt.clf()
fig = plt.figure(0, figsize=(10,10))
ax = fig.add_subplot(111)
xs = np.arange(1,13)-0.4
ax.bar(xs, [month[i] for i in range(1,13)])
ax.set_xticks(np.arange(1,13))
fig.savefig('pollfreq_month.png')
plt.clf()
fig = plt.figure(0, figsize=(20,10))
ax = fig.add_subplot(111)
xs = np.arange(1,54)-0.4
ax.bar(xs, [week[i] for i in range(1,54)])
ax.set_xticks(np.arange(1,54,5))
fig.savefig('pollfreq_week.png')
plt.clf()
fig = plt.figure(0, figsize=(10,10))
ax = fig.add_subplot(111)
xs = np.arange(1,32)-0.4
ax.bar(xs, [day[i] for i in range(1,32)])
ax.set_xticks(np.arange(1,32))
fig.savefig('pollfreq_day.png')
plt.clf()
def main():
fig_dir = 'chamber_chair_figs'
if not os.path.exists(fig_dir):
os.makedirs(fig_dir)
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
one_day = dt.timedelta(days=1)
for chair in range(1,350):
out_intervals = compute_intervals(s, chair)
q = s.query(ChamberAppointment).join(Member) \
.filter(ChamberAppointment.chair==chair) \
.order_by(ChamberAppointment.end_date.desc())
rows = OrderedDict()
for app in q:
if not app.member.id in rows:
rows[app.member.id] = [app]
else:
rows[app.member.id].append(app)
hatches = {'Tjänstgörande':'','Ledig':'x'}
colors = {'Riksdagsledamot': '#ff0000', 'Ersättare':'#aaaa22'}
fig = plt.figure(0,figsize=(100,len(rows)))
ax = fig.add_subplot(111)
y = 0
yticklabels = []
yticks = []
for (member_id, apps) in rows.items():
member = s.query(Member).filter(Member.id == member_id).one()
yticks.append(y+0.5)
yticklabels.append(repr(member))
for app in apps:
ax.barh(y,mdates.date2num(app.end_date+one_day)-mdates.date2num(app.start_date),
height=1.0,
left=app.start_date,
hatch=hatches[app.status],
color=colors[app.role],
linewidth=0)
y += 1
for i in out_intervals[1:]:
if len(i[2]) == 0: #empty chair
if i[1]-i[0] >= one_day:
ax.axvspan(i[0],i[1]+one_day, facecolor='#0000ff', alpha=0.5, linewidth=0)
ax.text(i[1]+one_day, y, 'Empty {}-{}'.format(i[0],i[1]),rotation=0)
else:
ax.axvspan(i[0],i[1]+one_day, facecolor='#00ffff', alpha=0.5, linewidth=0)
ax.text(i[1]+one_day, y, 'Empty one day {}'.format(i[0]),rotation=0)
if len(i[2]) > 1:
if i[1]-i[0] >= one_day:
ax.axvspan(i[0],i[1]+one_day, facecolor='#ff0000', alpha=0.5, linewidth=0)
ax.text(i[1]+one_day, y, 'n={}: Overlap {}-{}'.format(len(i[2]),i[0],i[1]),rotation=0)
else:
ax.axvspan(i[0],i[1]+one_day, facecolor='#ffff00', alpha=0.5, linewidth=0)
ax.text(i[1]+one_day, y, 'n={}: One day overlap {}'.format(len(i[2]),i[0]),rotation=0)
ax.set_ylim(0,y+1)
ax.yaxis.grid()
ax.set_yticks(yticks)
ax.set_yticklabels(yticklabels)
ax.xaxis_date()
ax.set_xlim(ax.get_xlim()[0],date(2019,1,1))
years = mdates.YearLocator() # every year
months = mdates.MonthLocator() # every month
yearsFmt = mdates.DateFormatter('%Y')
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.xaxis.set_minor_locator(months)
fig.autofmt_xdate()
plt.tight_layout()
fig.savefig('{}/{:03d}.pdf'.format(fig_dir,chair))
plt.clf()
