def main():
parameters, datapackage, resources = ingest()
jinja_env = get_jinja_env()
members = {}
committees = {}
for descriptor, resource in zip(datapackage["resources"], resources):
if descriptor["name"] == "mk_individual":
for member in resource:
mkId = member["mk_individual_id"]
members[mkId] = {
"mk_individual_id": mkId,
"first_name": member["mk_individual_first_name"],
"last_name": member["mk_individual_name"],
"photo": member["mk_individual_photo"],
"icon": getIcon(member["mk_individual_photo"]),
"position_url": POSITION_URL,
"ministry_url": MINISTRY_URL,
"faction_url": FACTION_URL,
"source_member_schema": descriptor["schema"],
"url": MEMBER_URL.format(member_id=mkId),
"source_member_row": member
}
elif descriptor['name'] == 'mk_individual_committees':
for position in resource:
committees_positions = members[
position['mk_individual_id']].setdefault('committees', [])
committee_chairperson_positions = members[
position['mk_individual_id']].setdefault(
'committee_chairperson', [])
if position['position_id'] == 41:
committee_chairperson_positions.append({
'knesset_num':
position['knesset'],
'committee_id':
position['committee_id'],
'committee_name':
position['committee_name'],
'start_date':
position['start_date'],
'finish_date':
position['finish_date']
})
else:
committees_positions.append({
'knesset_num':
position['knesset'],
'committee_id':
position['committee_id'],
'committee_name':
position['committee_name'],
'position_id':
position['position_id'],
'position_name':
position['position_name'],
'start_date':
position['start_date'],
'finish_date':
position['finish_date']
})
elif descriptor['name'] == 'mk_individual_faction_chairpersons':
for position in resource:
faction_chairpersons_positions = members[
position['mk_individual_id']].setdefault(
'faction_chairperson', [])
faction_chairpersons_positions.append({
'knesset_num':
position['knesset'],
'faction_id':
position['faction_id'],
'faction_name':
position['faction_name'],
'start_date':
position['start_date'],
'finish_date':
position['finish_date']
})
elif descriptor['name'] == 'mk_individual_factions':
for position in resource:
faction_positions = members[
position['mk_individual_id']].setdefault('factions', [])
faction_positions.append({
'knesset_num':
position['knesset'],
'faction_id':
position['faction_id'],
'faction_name':
position['faction_name'],
'start_date':
position['start_date'],
'finish_date':
position['finish_date']
})
elif descriptor['name'] == 'mk_individual_govministries':
for position in resource:
ministry_positions = members[
position['mk_individual_id']].setdefault(
'govministries', [])
ministry_positions.append({
'knesset_num':
position['knesset'],
'govministry_id':
position['govministry_id'],
'govministry_name':
position['govministry_name'],
'position_id':
position['position_id'],
'position_name':
position['position_name'],
'start_date':
position['start_date'],
'finish_date':
position['finish_date']
})
elif descriptor["name"] == "kns_committeesession":
for committeesession in resource:
# aggregate statistics only if there is a protocol and mks
if committeesession["text_parsed_filename"]:
knessetNum = committeesession["KnessetNum"]
if knessetNum not in committees:
committees[knessetNum] = 0
committees[knessetNum] += 1
for mkId in committeesession["attended_mk_individual_ids"]:
if mkId not in members:
continue
if isMember(members[mkId],
committeesession["StartDate"]):
if "counts" not in members[mkId]:
members[mkId]["counts"] = {}
if knessetNum not in members[mkId]["counts"]:
members[mkId]["counts"][knessetNum] = 0
members[mkId]["counts"][knessetNum] += 1
for member in members.values():
if not os.environ.get(
'DISABLE_MEMBER_PERCENTS') and "counts" in member:
for knesset, count in member["counts"].items():
percent = count / committees[knesset] * 100
if "percents" not in member:
member["percents"] = {}
member["percents"][knesset] = int(percent)
build_template(jinja_env, "member_detail.html",
get_member_context(member),
MEMBER_URL.format(member_id=member["mk_individual_id"]))
if os.environ.get("SKIP_STATIC") != "1":
subprocess.check_call(
["mkdir", "-p", "../../data/committees/dist/dist"])
subprocess.check_call(
["cp", "-rf", "static", "../../data/committees/dist/dist/"])
spew(dict(datapackage, resources=[]), [], {})
update_hash_callback(''.join(
(str(mk_individuals[mk_id])
for mk_id in meeting["attended_mk_individual_ids"])).encode())
update_hash_callback(str(kns_committees[meeting["CommitteeID"]]).encode())
update_hash_callback(str(meeting).encode())
for meeting in next(resources):
m = HASH_FILES_OBJECT.copy()
update_meeting_hash(meeting, m.update)
build_template(jinja_env,
"committeemeeting_detail.html",
lambda: get_meeting_context(
meeting,
dict(mk_individuals=mk_individuals,
kns_committees=kns_committees,
meetings_descriptor=meetings_descriptor,
kns_committee_descriptor=kns_committee_descriptor,
meeting_stats=meeting_stats)),
get_meeting_path(meeting),
with_hash=m.hexdigest())
stats["meetings"] += 1
def get_stats_resource():
for meeting_id, stats in meeting_stats.items():
stats["CommitteeSessionID"] = meeting_id
yield stats
spew(
def main():
parameters, datapackage, resources = ingest()
jinja_env = get_jinja_env()
members = {}
factions = {}
knessets = {}
counts = {}
for descriptor, resource in zip(datapackage["resources"], resources):
if descriptor["name"] == "members":
for member in resource:
members[member["mk_individual_id"]] = member
elif descriptor["name"] == "positions":
for position in resource:
if position["object_type"] == "faction":
factions[position["object_id"]] = {
"faction_num": position["object_id"],
"faction_name": position["object_name"],
"mks": []
}
for id in position["mk_individual_ids"]:
factions[position["object_id"]]["mks"].append(
members[id])
elif descriptor["name"] == "knessets":
for knesset in resource:
knessets[knesset["KnessetNum"]] = []
for id in knesset["faction"]:
knessets[knesset["KnessetNum"]].append(factions[id])
for knesset_num, factions in knessets.items():
mks = set()
build_template(
jinja_env, "factions_index.html",
get_context({
"knesset_num":
knesset_num,
"factions":
factions,
"faction_url":
FACTION_URL.format(knesset_num=knesset_num,
faction_id="{faction_id}"),
"member_url":
MEMBER_URL
}), FACTION_HOME_URL.format(knesset_num=knesset_num))
for faction in factions:
if not faction['faction_name']:
continue
for mk in faction['mks']:
mks.add(mk['mk_individual_id'])
build_template(
jinja_env, "faction_detail.html",
get_context({
"knesset_num":
knesset_num,
"faction_num":
faction["faction_num"],
"faction_name":
faction["faction_name"],
"mks":
faction["mks"],
"faction_home_url":
FACTION_HOME_URL.format(knesset_num=knesset_num),
"member_url":
MEMBER_URL
}),
FACTION_URL.format(knesset_num=knesset_num,
faction_id=faction["faction_num"]))
counts[knesset_num] = {'factions': len(factions), 'mks': len(mks)}
build_template(
jinja_env, "members_index.html",
get_context({
"keys": sorted(counts, reverse=True),
"knessets": counts,
"url": FACTION_HOME_URL
}), MEMBERS_HOME_URL)
if os.environ.get("SKIP_STATIC") != "1":
subprocess.check_call(
["mkdir", "-p", "../../data/committees/dist/dist"])
subprocess.check_call(
["cp", "-rf", "static", "../../data/committees/dist/dist/"])
spew(dict(datapackage, resources=[]), [], {})
return get_context({
"knesset_nums":
sorted(knesset_nums(), key=lambda k: k["num"], reverse=True)
})
jinja_env = get_jinja_env()
stats["built_committees"] = 0
stats["failed_committees"] = 0
knesset_num_committees = {}
for kns_committee in kns_committees.values():
knesset_num_committees.setdefault(kns_committee["KnessetNum"],
[]).append(kns_committee)
build_template(
jinja_env, "committee_detail.html",
get_committee_context(kns_committee),
COMMITTEE_DETAIL_URL.format(committee_id=kns_committee["CommitteeID"]))
stats["built_committees"] += 1
stats["built_knesset_nums"] = 0
stats["failed_knesset_nums"] = 0
for knesset_num, kns_committees in knesset_num_committees.items():
build_template(jinja_env, "committee_list.html",
get_committee_list_context(kns_committees, knesset_num),
COMMITTEE_LIST_KNESSET_URL.format(num=knesset_num))
stats["built_knesset_nums"] += 1
build_template(jinja_env, "committees_index.html",
get_committee_index_context(knesset_num_committees),
COMMITTEES_INDEX_URL)
stats["built index"] = 1
committee,
"attended_mks":
attended_mks,
})
meeting_stats[meeting["CommitteeSessionID"]] = {
"num_speech_parts": len(speech_parts_list)
}
return context
for meeting in next(resources):
m = HASH_FILES_OBJECT.copy()
update_meeting_hash(meeting, m.update)
build_template(jinja_env,
"committeemeeting_detail.html",
lambda: get_meeting_context(meeting),
get_meeting_path(meeting),
with_hash=m.hexdigest())
stats["meetings"] += 1
def get_stats_resource():
for meeting_id, stats in meeting_stats.items():
stats["CommitteeSessionID"] = meeting_id
yield stats
spew(
dict(datapackage,
resources=[{
PROP_STREAMING: True,