def cosigning_matrix(partyA):
ds = MongoDBDatastore()
mongodb = ds.get_mongodb_database()
mongo_collection = mongodb.party_cosigning_matrix
record = mongo_collection.find_one({"partyA": party_abbr})
if record:
del record['_id']
return jsonify(record)
else:
return render_template('404.html'), 404
def proposals_main(government):
mdb = MongoDBDatastore()
mongodb = mdb.get_mongodb_database()
mongo_collection = mongodb.proposals_main
record = mongo_collection.find_one({"government": government})
if record is None:
raise KeyError
del record['_id']
return record
def multiparties_detail(government):
mdb = MongoDBDatastore()
mongodb = mdb.get_mongodb_database()
mongo_collection = mongodb.proposals_multiparties_detail
print(mongo_collection)
record = mongo_collection.find_one({"government": government})
if record is None:
raise KeyError
del record['_id']
return record
def committee_detail(government, id):
mdb = MongoDBDatastore()
mongodb = mdb.get_mongodb_database()
mongo_collection = mongodb.proposals_committee_detail
record = mongo_collection.find_one({
"government": government,
'committee_id': repr(id)
})
if record is None:
raise KeyError
del record['_id']
return record
def party_bias(partyA, partyB):
s = db.session
A_id = s.query(Party.id).filter(Party.abbr == partyA).one()[0]
B_id = s.query(Party.id).filter(Party.abbr == partyB).one()[0]
ds = MongoDBDatastore()
mongodb = ds.get_mongodb_database()
mongo_collection = mongodb.party_covoting
record = mongo_collection.find_one({"partyA": A_id, "partyB": B_id})
if record:
del record['_id']
return jsonify(record)
else:
return render_template('404.html'), 404
def main():
d = scb.get_request(URL, QUERY)
out_data = scb.NestedDefaultdict()
for entry in d["data"]:
val = entry["values"][0]
if val == scb.na_val:
continue
val = float(val) / 100
party_abbr = entry["key"][1].lower()
year = int(entry["key"][2])
out_data[party_abbr][year] = val
ds = MongoDBDatastore()
ds.store_object(out_data.to_dict(), "election_results")
def main():
d = scb.get_request(URL, QUERY)
out_data = scb.NestedDefaultdict()
for entry in d["data"]:
val = entry["values"][0]
if val == scb.na_val:
continue
val = float(val) / 100
party_abbr = entry["key"][0].lower()
year, month = map(int, entry["key"][1].split('M'))
day = 1
date = dt.datetime(year, month, day)
out_data[party_abbr][date] = val
ds = MongoDBDatastore()
ds.store_object(out_data.to_dict(), "scb_polls")
def cosigning_timeseries():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
party_metadata = dict()
for party in s.query(Party):
party_metadata[party.id] = dict(abbr=party.abbr,
name=party.name,
ordering=party_ordering[party.abbr])
committee_metadata = dict()
for c in s.query(Committee):
committee_metadata[c.id] = dict(abbr=c.abbr, name=c.name)
committee_metadata[0] = dict(abbr="Alla", name="Alla utskott")
basequery = s.query(MemberProposal) \
.options(joinedload(MemberProposal.points).joinedload(ProposalPoint.committee_report).joinedload(CommitteeReport.committee)) \
.filter(MemberProposal.subtype != 'Enskild motion')
riksmoten = [
'{:04d}/{:02d}'.format(y, y - 2000 + 1) for y in range(2002, 2014)
]
output = dict()
num_missing_committee_report = 0
num_missing_committee_report_committee = 0
matrix = dict()
for (rm_idx, rm) in enumerate(riksmoten):
print("RM: {}".format(rm))
for doc in basequery.filter(MemberProposal.session == rm):
if not doc.signatories or len(doc.signatories) == 1:
continue
signing_parties = sorted(
list(
set(m.party.id for m in doc.signatories
if not m.party.abbr == "-")))
if len(signing_parties) <= 1:
continue
parties_key = repr(signing_parties)
for point in doc.points:
if not point.committee_report:
num_missing_committee_report += 1
continue
if not point.committee_report.committee:
num_missing_committee_report_committee += 1
continue
committee = point.committee_report.committee
# compute party matrix data
for (p1, p2) in permutations(signing_parties, 2):
if not p1 in matrix:
matrix[p1] = dict()
for committee_key in [0, committee.id]:
if not committee_key in matrix[p1]:
matrix[p1][committee_key] = dict()
if not p2 in matrix[p1][committee_key]:
matrix[p1][committee_key][p2] = dict(
values=[0 for rm in riksmoten],
abbr=party_metadata[p2]["abbr"],
name=party_metadata[p2]["name"],
id=p2)
matrix[p1][committee_key][p2]["values"][rm_idx] += 1
# compute timeseries data
for committee_key in [0, committee.id]:
if not committee_key in output:
output[committee_key] = dict()
if not parties_key in output[committee_key]:
output[committee_key][parties_key] = \
dict(
values = [0 for rm in riksmoten],
abbr = ' + '.join(party_metadata[p_id]["abbr"] for p_id in signing_parties),
name = ' + '.join(party_metadata[p_id]["name"] for p_id in signing_parties),
num_parties = len(signing_parties)
)
output[committee_key][parties_key]["values"][rm_idx] += 1
ds = MongoDBDatastore()
output_timeseries_top = dict(
t=riksmoten,
committees=[
dict(abbr=committee_metadata[c_id]["abbr"],
name=committee_metadata[c_id]["name"],
id=c_id,
series=[[
dict(abbr=p_dict["abbr"],
name=p_dict["name"],
num_parties=p_dict["num_parties"],
value=p_dict["values"][rm_idx])
for (p_key, p_dict) in series.items()
if p_dict["values"][rm_idx] > 0
] for (rm_idx, rm) in enumerate(riksmoten)])
for (c_id, series) in output.items()
])
ds.store_object(output_timeseries_top, "party_cosigning_timeseries")
mongodb = ds.get_mongodb_database()
mongo_collection = mongodb.party_cosigning_matrix
mongo_collection.ensure_index([("partyA", ASCENDING)], unique=True)
for (p1, matrix_p1) in matrix.items():
mongo_partyA = party_metadata[p1]["abbr"]
output_party_matrix_top = dict(
partyA=mongo_partyA,
t=riksmoten,
committees=[
dict(abbr=committee_metadata[c_id]["abbr"],
name=committee_metadata[c_id]["name"],
id=c_id,
parties=list(committee_dict.values()))
for (c_id, committee_dict) in matrix_p1.items()
])
mongo_collection.update(dict(partyA=mongo_partyA),
output_party_matrix_top,
upsert=True)
print(
"missing committee_report: {}, committee_reports missing committee's: {}"
.format(num_missing_committee_report,
num_missing_committee_report_committee))
def timeseries():
ds = MongoDBDatastore()
cosigning_data = ds.get_object("party_cosigning_timeseries")
return jsonify(cosigning_data)
def main():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
party_metadata = dict()
for party in s.query(Party):
party_metadata[party.id] = dict(
abbr=party.abbr,
name=party.name,
ordering=party_ordering[party.abbr]
)
committee_metadata = dict()
for c in s.query(Committee):
committee_metadata[c.id] = dict(
abbr=c.abbr,
name=c.name
)
result_metadata = dict()
result_metadata['bifall'] = dict(
name= 'Bifall',
ordering = 2
)
result_metadata['delvis bifall'] = dict(
name= 'Delvis bifall',
ordering = 1
)
result_metadata['avslag'] = dict(
name= 'Avslag',
ordering = 0
)
mdb = MongoDBDatastore()
mongodb = mdb.get_mongodb_database()
mongo_collection = mongodb.proposals_main
mongo_collection.ensure_index([("government", ASCENDING)], unique=True)
mongo_collection_party_detail = mongodb.proposals_party_detail
mongo_collection_party_detail.ensure_index([("government", ASCENDING),("party_id", ASCENDING)], unique=True)
mongo_collection_committee_detail = mongodb.proposals_committee_detail
mongo_collection_committee_detail.ensure_index([("government", ASCENDING),("committee_id", ASCENDING)], unique=True)
mongo_collection_ministries_detail = mongodb.proposals_ministries_detail
mongo_collection_ministries_detail.ensure_index([("government", ASCENDING)], unique=True)
mongo_collection_multiparties_detail = mongodb.proposals_multiparties_detail
mongo_collection_multiparties_detail.ensure_index([("government", ASCENDING)], unique=True)
mongo_collection_members_detail = mongodb.proposals_members_detail
mongo_collection_members_detail.ensure_index([("government", ASCENDING)], unique=True)
basequery = s.query(Proposal).options( \
joinedload(Proposal.points).joinedload(ProposalPoint.committee_report).joinedload(CommitteeReport.committee)) \
.order_by(Proposal.published.desc())
governments = [
("persson3",
dict(title="Person III", years="2002-2006", party="S"),
basequery.filter(MemberProposal.session.in_(['2002/03', '2003/04', '2004/05', '2005/06']))
),
("reinfeldt1",
dict(title="Reinfeldt I", years="2006-2010", party="M"),
basequery.filter(MemberProposal.session.in_(['2006/07', '2007/08', '2008/09', '2009/10']))
),
("reinfeldt2",
dict(title="Reinfeldt II", years="2010-2014", party="M"),
basequery.filter(MemberProposal.session.in_(['2010/11', '2011/12', '2012/13', '2013/14']))
)
]
num_missing_committee_report = 0
num_missing_committee_report_committee = 0
num_unknown_result = 0
for (name, government_metadata, query) in governments:
print("Government: {}".format(name))
parties_tree = Tree()
multiparties_tree = Tree()
ministries_tree = Tree()
members_tree = Tree()
committees_set = set()
results_set = set()
for doc in query:
if hasattr(doc, 'ministry_id'):
origin_tree = ministries_tree
origin_key = "Alla departement"
else:
if not doc.signatories or len(doc.signatories) == 0:
continue
signing_parties = sorted(list(set( m.party.id for m in doc.signatories)))
if doc.subtype == 'Enskild motion':
origin_tree = members_tree
origin_key = "Alla enskilda motioner"
elif len(signing_parties) == 1:
origin_tree = parties_tree
origin_key = signing_parties[0]
else: #multiparty signature
origin_tree = multiparties_tree
origin_key = "Alla Flerpartiförslag"
for point in doc.points:
if not point.committee_report:
print('Missing committee_report: {}, {}'.format(doc, point.number))
num_missing_committee_report += 1
continue
if not point.committee_report.committee:
print('Missing committee_report.committee: {}, {}'.format(doc, point.number))
num_missing_committee_report_committee += 1
continue
committee = point.committee_report.committee
result = point.decision
committee_key = committee.id
result_key = result.lower()
if not result_key in result_metadata.keys():
num_unknown_result += 1
print("Unknown result: {} {}".format(result_key, doc))
continue
if not result_key in origin_tree[origin_key][committee_key]:
origin_tree[origin_key][committee_key][result_key] = list()
origin_tree[origin_key][committee_key][result_key].append((doc.id, point.number))
committees_set.add(committee_key)
results_set.add(result_key)
def party_sort_key(p):
return party_metadata[p]['ordering']
def result_sort_key(r):
return result_metadata[r]['ordering']
parties = list(parties_tree.keys())
parties.sort(key=party_sort_key)
party_idc = {p:k for (k,p) in enumerate(parties)}
multiparties = list(multiparties_tree.keys())
multiparty_idc = {p:k for (k,p) in enumerate(multiparties)}
ministries = list(ministries_tree.keys())
ministry_idc = {p:k for (k,p) in enumerate(ministries)}
members = list(members_tree.keys())
member_idc = {p:k for (k,p) in enumerate(members)}
committees = list(committees_set)
committee_idc = {c:k for (k,c) in enumerate(committees)}
results = list(results_set)
results.sort(key=result_sort_key)
result_idc = {r:k for (k,r) in enumerate(results)}
nodes = list()
node_group_members = dict(
title="Enskilda\nledamöters\nmotioner",
items=[dict(detail="members", title=p) for p in members],
label = -1)
node_group_party = dict(
title="Partiernas\nmotioner",
items=[dict(detail="party", party_id=p, title=party_metadata[p]['name'],abbr=party_metadata[p]['abbr']) for p in parties],
label = -1)
node_group_multiparty = dict(
title="Flerparti-\nmotioner",
items=[dict(detail="multiparties", title=p) for p in multiparties],
label = -1)
node_group_government = dict(
title="Regeringens\npropositioner",
items=[dict(detail="government", title=p) for p in ministries],
label = -1)
node_group_committee = dict(
title="Utskotten",
items=[dict(detail="committee",committee_id=p, title=committee_metadata[p]['name'],abbr=committee_metadata[p]['abbr']) for p in committees],
label = 0)
node_group_results = dict(
title="Beslut",
items=[dict(detail="beslut", result_id=r, title=result_metadata[r]['name']) for r in results],
label = 0)
nodes = [
dict(x=0.0, title='1. Förslag kommer från\nriksdagens ledamöter\noch regeringen', items=[node_group_members, node_group_party, node_group_multiparty, node_group_government]),
dict(x=0.5, title='2. De bereds i ett\nav utskotten.', items=[node_group_committee]),
dict(x=1.0, title='3. Beslut i\nriksdagens kammare.', items=[node_group_results])
]
flows = list()
for member_key in members_tree:
for committee_key in members_tree[member_key]:
for (result_key, point_ids) in members_tree[member_key][committee_key].items():
member_addr = (0,0,member_idc[member_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[member_addr, committee_addr, result_addr], magnitude=len(point_ids)))
for party_key in parties_tree:
for committee_key in parties_tree[party_key]:
for (result_key, point_ids) in parties_tree[party_key][committee_key].items():
party_addr = (0,1,party_idc[party_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[party_addr, committee_addr, result_addr], magnitude=len(point_ids)))
for multiparty_key in multiparties_tree:
for committee_key in multiparties_tree[multiparty_key]:
for (result_key, point_ids) in multiparties_tree[multiparty_key][committee_key].items():
multiparty_addr = (0,2,multiparty_idc[multiparty_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[multiparty_addr, committee_addr, result_addr], magnitude=len(point_ids)))
for ministry_key in ministries_tree:
for committee_key in ministries_tree[ministry_key]:
for (result_key, point_ids) in ministries_tree[ministry_key][committee_key].items():
ministry_addr = (0,3,ministry_idc[ministry_key])
committee_addr = (1,0,committee_idc[committee_key])
result_addr = (2,0,result_idc[result_key])
flows.append(dict(path=[ministry_addr, committee_addr, result_addr], magnitude=len(point_ids)))
def parse_title(s,dbs):
r = re.sub(r'med anledning av', 'm.a.a.', s)
r = re.sub(r'([0-9]{4}/[0-9]{2}):(\S{,7})', lambda m: '<a href="{}">{}</a>'.format(riksdagen.url_from_code(m.group(1),m.group(2),dbs),m.group(0)), r)
return r
def author_string(doc):
if len(doc.signatories)>2:
return '{} och {} till'.format(repr(doc.signatories[0]),len(doc.signatories)-1)
else:
return ' och '.join(map(repr,doc.signatories))
def shortest_number_string(l):
items = []
i = 0
on_streak = False
while i < len(l):
if not on_streak:
start = i
if len(l) == i+1 or l[i+1] != l[i]+1:
if start == i:
items.append(repr(l[start]))
else:
if i-start == 1:
items.append('{},{}'.format(repr(l[start]),repr(l[i])))
else:
items.append('{}-{}'.format(repr(l[start]),repr(l[i])))
on_streak = False
else:
on_streak = True
i += 1
return ', '.join(items)
def documents_from_points(decision_sets):
result = list()
prevdocid = 0
row = None
for (decision, ids) in decision_sets:
for (docid, number) in ids:
if not (docid == prevdocid):
doc = s.query(Document).get(docid)
parsed_title = parse_title(doc.title,s)
authors = author_string(doc)
title_html = '<span class="authors">{}</span><span class="title">{}</span>'.format(authors, parsed_title)
row = dict(
decision=decision,
title=title_html,
unique_code=doc.unique_code(),
url=riksdagen.url_from_dokid(doc.dok_id),
numbers='')
numbers = []
result.append(row)
numbers.append(number)
numbers.sort()
row['numbers'] = shortest_number_string(numbers)
prevdocid = docid
return result
def documents_from_prop(decision_sets):
result = list()
prevdocid = 0
row = None
for (decision, ids) in decision_sets:
for (docid, number) in ids:
if not (docid == prevdocid):
doc = s.query(Document).get(docid)
parsed_title = parse_title(doc.title,s)
authors = "?"
if not doc.ministry is None:
authors = doc.ministry.name
title_html = '<span class="authors">{}</span><span class="title">{}</span>'.format(authors, parsed_title)
row = dict(
decision=decision,
title=title_html,
unique_code=doc.unique_code(),
url=riksdagen.url_from_dokid(doc.dok_id),
numbers='')
numbers = []
result.append(row)
numbers.append(number)
numbers.sort()
row['numbers'] = shortest_number_string(numbers)
prevdocid = docid
return result
def committee_detail_for_tree(tree, doctype):
total_bifall = 0
total_delvis_bifall = 0
total_avslag = 0
committee_results = []
for c_id in committees:
if c_id not in tree:
continue
bifall_ids = tree[c_id].get('bifall',[])
delvis_bifall_ids = tree[c_id].get('delvis bifall',[])
avslag_ids = tree[c_id].get('avslag',[])
num_bifall = len(bifall_ids)
num_delvis_bifall = len(delvis_bifall_ids)
num_avslag = len(avslag_ids)
total_bifall += num_bifall
total_delvis_bifall += num_delvis_bifall
total_avslag += num_avslag
all_ids = (("Bifall", bifall_ids), ("Delvis bifall", delvis_bifall_ids), ("Avslag", avslag_ids))
doc_function = documents_from_points
if doctype == 'government':
all_ids = all_ids[::-1]
doc_function = documents_from_prop
committee_results.append({
'name': committee_metadata[c_id]['name'],
'abbr': committee_metadata[c_id]['abbr'],
'bifall': num_bifall,
'delvis_bifall': num_delvis_bifall,
'avslag': num_avslag,
'documents': doc_function(all_ids)
})
return dict(
committee_results = committee_results,
total_bifall = total_bifall,
total_delvis_bifall = total_delvis_bifall,
total_avslag = total_avslag
)
def origin_detail_for_tree(tree, doctype):
origin_results = []
bifall_ids = tree[c_id].get('bifall',[])
delvis_bifall_ids = tree[c_id].get('delvis bifall',[])
avslag_ids = tree[c_id].get('avslag',[])
num_bifall = len(bifall_ids)
num_delvis_bifall = len(delvis_bifall_ids)
num_avslag = len(avslag_ids)
all_ids = (("Bifall", bifall_ids), ("Delvis bifall", delvis_bifall_ids), ("Avslag", avslag_ids))
doc_function = documents_from_points
if doctype == 'government':
all_ids = all_ids[::-1]
doc_function = documents_from_prop
return {
'bifall': num_bifall,
'delvis_bifall': num_delvis_bifall,
'avslag': num_avslag,
'documents': doc_function(all_ids)
}
print('Generating party_detail')
party_detail = dict();
for p_id in parties:
committee_detail = committee_detail_for_tree(parties_tree[p_id], 'motion')
committee_detail['party'] = party_metadata[p_id]
party_detail[repr(p_id)] = committee_detail
print('Generating ministries_detail')
ministries_detail = committee_detail_for_tree(ministries_tree[ministries[0]], 'government')
print('Generating members_detail')
members_detail = committee_detail_for_tree(members_tree[members[0]],'motion')
print('Generating multiparties_detail')
multiparties_detail = committee_detail_for_tree(multiparties_tree[multiparties[0]],'motion')
print('Generating committee_detail')
committee_detail = dict();
for c_id in committees:
total_bifall = 0
total_delvis_bifall = 0
total_avslag = 0
origin_results = []
# government proposals first
ret = origin_detail_for_tree(ministries_tree[ministries[0]], 'government')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name='Regeringens propositioner',
abbr='R'
))
origin_results.append(ret)
# then members
ret = origin_detail_for_tree(members_tree[members[0]], 'motion')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name='Enskilda ledamöters motioner',
abbr='elm'
))
origin_results.append(ret)
# then multiparties
ret = origin_detail_for_tree(multiparties_tree[multiparties[0]], 'motion')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name='Samarbetsförslag',
abbr='sf'
))
origin_results.append(ret)
# finally parties
for p_id in parties:
if c_id not in parties_tree[p_id]:
continue
ret = origin_detail_for_tree(parties_tree[p_id], 'motion')
total_bifall += ret['bifall']
total_delvis_bifall += ret['delvis_bifall']
total_avslag += ret['avslag']
ret.update(dict(
name=party_metadata[p_id]['name'],
abbr=party_metadata[p_id]['abbr']
))
origin_results.append(ret)
committee_detail[repr(c_id)] = dict(
committee = committee_metadata[c_id],
origin_results = origin_results,
total_bifall = total_bifall,
total_delvis_bifall = total_delvis_bifall,
total_avslag = total_avslag
)
output_top = dict(
government=name,
nodes = nodes,
flows = flows
)
output_top.update(government_metadata)
mongo_collection.update(dict(government=name), output_top, upsert=True)
ministries_detail.update(dict(government=name))
mongo_collection_ministries_detail.update(dict(government=name), ministries_detail, upsert=True)
members_detail.update(dict(government=name))
mongo_collection_members_detail.update(dict(government=name), members_detail, upsert=True)
multiparties_detail.update(dict(government=name))
mongo_collection_multiparties_detail.update(dict(government=name), multiparties_detail, upsert=True)
for (p_id, d) in party_detail.items():
d.update(dict(
government=name,party_id = p_id,
))
mongo_collection_party_detail.update(dict(government=name,party_id = p_id), d, upsert=True)
for (c_id, d) in committee_detail.items():
d.update(dict(
government=name,committee_id = c_id,
))
mongo_collection_committee_detail.update(dict(government=name,committee_id = c_id), d, upsert=True)
print("missing committee_report: {}\ncommittee_reports missing committee's: {}\nunknown result: {}".format(num_missing_committee_report,num_missing_committee_report_committee,num_unknown_result))
def main():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
# Enumerate all searchable objects and arrange in groups
members = list()
member_indices = dict()
for (idx, member) in enumerate(s.query(Member)):
members.append(
dict(title=member.first_name + " " + member.last_name + " (" +
member.party.abbr + ")",
img_url=member.image_url_sm,
url='/' + member.url_name))
member_indices[member.id] = idx
parties = list()
party_indices = dict()
for (idx, party) in enumerate(s.query(Party)):
parties.append(
dict(title=party.name,
img_url='/static/img/parties/' + party.abbr.lower() + '.png',
url='/' + party.abbr.lower()))
party_indices[party.id] = idx
output_groups = [
dict(title="Ledamöter", objects=members),
dict(title="Partier", objects=parties)
]
group_indices = dict(member=0, party=1)
# Enumerate all strings to match and couple them to objects (primary and secondary hits)
keywords = list()
for member in s.query(Member):
normalized_string = (member.first_name + " " +
member.last_name).lower()
keywords.append(
dict(string=normalized_string,
primary=(group_indices["member"], member_indices[member.id]),
secondaries=[(group_indices["party"],
party_indices[member.party.id])]))
for party in s.query(Party):
normalized_string = (party.name).lower()
keywords.append(
dict(string=normalized_string,
primary=(group_indices["party"], party_indices[party.id]),
secondaries=[(group_indices["member"], member_indices[m.id])
for m in party.members[0:2]]))
# Generate reverse table of all trigrams to match, such that we can ask:
# which keywords contain this trigram (and how many)
trigrams_dict = dict()
for (keyword_idx, keyword) in enumerate(keywords):
grams = Counter(
map(lambda t: ''.join(t),
zip(*[keyword["string"][k:] for k in range(3)])))
for (gram, count) in grams.items():
if not gram in trigrams_dict:
trigrams_dict[gram] = list()
trigrams_dict[gram].append((keyword_idx, count))
print('# unique trigrams: {}'.format(len(trigrams_dict)))
output_top = dict(groups=output_groups, keywords=keywords)
ds = MongoDBDatastore()
ds.store_object(output_top, "search")
municipalities[m][y][sort_special[p]] = dp[m]
else:
if not math.isnan(dp[m]):
municipalities[m][y]['total_votes'] += dp[m]
totals = {}
for p, dp in party_sums.items():
for y, val in dp.items():
if y not in totals:
totals[y] = 0
totals[y] += val
# Set municipalities relative to number of votes (leave NaNs)
for y, dy in elec_dict.items():
for p, dp in dy.items():
for m in dp.keys():
try:
if not math.isnan(dp[m]):
dp[m] = dp[m] / municipalities[m][y]['total_votes']
except ZeroDivisionError as e:
print("Division by zero: municipality {}, year {}.".format(
m, y))
print("Storing in MongoDB.")
ds = MongoDBDatastore()
ds.store_object(elec_dict, "election_municipalities")
ds.store_object(municipalities, "election_municipality_sums")
ds.store_object(party_sums, "election_party_sums")
ds.store_object(totals, "election_totals")
def main():
engine = create_engine(pg_utils.engine_url())
session = sessionmaker(bind=engine)
s = session()
riksmoten = ['{:04d}/{:02d}'.format(y,y-2000+1) for y in range(2002,2014)]
parties = s.query(Party) \
.filter(Party.abbr != "-") \
.order_by(Party.id)
v1 = aliased(PartyVote)
v2 = aliased(PartyVote)
v3 = aliased(PartyVote)
mdb = MongoDBDatastore()
mongodb = mdb.get_mongodb_database()
mongo_collection = mongodb.party_covoting
mongo_collection.ensure_index([("partyA",ASCENDING),("partyB",ASCENDING)],unique=True)
for (partyA, partyB) in combinations(parties, 2):
conflicting_votes = s.query(v1, v2) \
.filter(v1.polled_point_id == v2.polled_point_id) \
.filter(v1.party_id == partyA.id) \
.filter(v2.party_id == partyB.id) \
.filter(or_(v1.vote_option == 'Ja', v1.vote_option == 'Nej')) \
.filter(or_(v2.vote_option == 'Ja', v2.vote_option == 'Nej')) \
.filter(v1.vote_option != v2.vote_option)
num_conflicting = conflicting_votes.count()
if num_conflicting < 1000:
continue
print('================================')
print('{} vs {}: {} conflicting votes'.format(partyA.abbr,partyB.abbr, num_conflicting))
output_parties = list()
for party in parties:
print('{}...'.format(party.abbr))
agree_query = s.query(v1, v2, v3, PolledPoint,CommitteeReport) \
.filter(CommitteeReport.id == PolledPoint.committee_report_id) \
.filter(PolledPoint.id == v1.polled_point_id) \
.filter(v1.polled_point_id == v2.polled_point_id) \
.filter(v1.party_id == partyA.id) \
.filter(v2.party_id == partyB.id) \
.filter(or_(v1.vote_option == 'Ja', v1.vote_option == 'Nej')) \
.filter(or_(v2.vote_option == 'Ja', v2.vote_option == 'Nej')) \
.filter(v1.vote_option != v2.vote_option) \
.filter(v3.polled_point_id == v2.polled_point_id) \
.filter(v3.party_id == party.id)
party_bias = list()
for (rm_idx, rm) in enumerate(riksmoten):
agree_query_interval = agree_query.filter(CommitteeReport.session == rm)
agreeA = agree_query_interval.filter(v3.vote_option == v1.vote_option).count()
agreeB = agree_query_interval.filter(v3.vote_option == v2.vote_option).count()
print(rm,agreeA,agreeB)
if agreeA + agreeB > 0:
party_bias.append( float(agreeB - agreeA)/float(agreeA + agreeB))
else:
party_bias.append(None)
if all(x is None for x in party_bias):
continue
output_parties.append(dict(
party_id=party.id,
party_abbr=party.abbr,
party_name=party.name,
values=party_bias))
print('Dumping to MongoDB.')
output_parties_reverse = deepcopy(output_parties)
for item in output_parties_reverse:
item['values'] = [x if x is None else -x for x in item['values']]
output_top = dict(partyA = partyA.id, partyB = partyB.id, series = output_parties,labels=riksmoten)
output_top_reverse = dict(partyA = partyB.id, partyB = partyA.id, series = output_parties_reverse, labels=riksmoten)
mongo_collection.update(dict(partyA = partyA.id, partyB = partyB.id), output_top, upsert=True)
mongo_collection.update(dict(partyA = partyB.id, partyB = partyA.id), output_top_reverse, upsert=True)
# coding=utf-8
from demokratikollen.www.app.helpers.db import db
from sqlalchemy import func
from demokratikollen.www.app.helpers.cache import cache
from demokratikollen.core.db_structure import Party, Group, GroupAppointment, Member
from demokratikollen.core.utils.mongodb import MongoDBDatastore
import math
from demokratikollen.data.other.election_dates import ELECTION_DATES
from demokratikollen.data.other.party_leaders import PARTY_LEADERS
import datetime as dt
import calendar
import operator
mdb = MongoDBDatastore()
def party_comparator(p1):
party_sort = {
'-': 0,
'v': 1,
's': 2,
'mp': 3,
'sd': 4,
'nyd': 5,
'c': 6,
'fp': 7,
'l': 7,
'kd': 8,
'm': 9
def timeseries():
ds = MongoDBDatastore()
return jsonify(ds.get_object("search"))
"code": "ContentsCode",
"selection": {
"filter": "item",
"values": ["ME0201AV"]
}
}],
"response": {
"format": "json"
}
}
url = "http://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0201/ME0201B/Partisympati17"
d = scb.get_request(url, query)
data = scb.NestedDefaultdict()
key_order = scb.best_party_gender_key_order
for entry in d["data"]:
val = entry["values"][0]
val = float('NaN') if val == scb.na_val else float(val) / 100
keys = scb.translate_keys(entry["key"])
keys_d = dict(zip(key_order, keys))
t = keys_d["Tid"]
p = keys_d["Partisympati"]
data[p][t] = val
ds = MongoDBDatastore()
ds.store_object(data.to_dict(), "best_party_time")
"filter": "item",
"values": ["ME0201AV"]
}
}],
"response": {
"format": "json"
}
}
url = "http://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0201/ME0201B/Partisympati17"
d = scb.get_request(url, query)
data = scb.NestedDefaultdict()
key_order = scb.best_party_gender_key_order
for entry in d["data"]:
val = entry["values"][0]
val = float('NaN') if val == scb.na_val else float(val) / 100
keys = scb.translate_keys(entry["key"])
keys_d = dict(zip(key_order, keys))
t = keys_d["Tid"]
p = keys_d["Partisympati"]
g = keys_d["Kon"]
e = keys_d["UtbNivaSUN2000"]
data[t][p][e] = val
ds = MongoDBDatastore()
ds.store_object(data.to_dict(), "best_party_education")