def load_committee_history(committee_id, cycle=None):
filters = {"per_page": 1}
if not cycle:
# if no cycle parameter given,
# (1.1)call committee/{committee_id}/history/ under tag:committee
# set cycle = fallback_cycle
path = "/committee/" + committee_id + "/history/"
committee = api_caller.load_first_row_data(path, **filters)
if committee is None:
raise Http404()
cycle = committee.get("last_cycle_has_financial")
if not cycle:
# when committees only file F1.fallback_cycle = null
# set cycle = last_cycle_has_activity
cycle = committee.get("last_cycle_has_activity")
else:
# (1.2)call committee/{committee_id}/history/{cycle}/
# under tag:committee
path = "/committee/" + committee_id + "/history/" + str(cycle)
committee = api_caller.load_first_row_data(path, **filters)
if committee is None:
raise Http404()
# (2)call committee/{committee_id}/candidates/history/{cycle}
# under: candidate, get all candidates associated with that commitee
path = "/committee/" + committee_id + "/candidates/history/" + str(cycle)
all_candidates = api_caller.load_endpoint_results(path,
election_full=False)
# clean cycles_has_activity, remove 'None' value in cycles_has_activity
committee["cycles_has_activity"] = list(
filter(None, committee.get("cycles_has_activity")))
return committee, all_candidates, cycle
def load_most_recent_candidate(candidate_id):
"""
Get most recent candidate information
"""
path = "/candidate/" + candidate_id + "/history/"
filters = {"per_page": 1}
return api_caller.load_first_row_data(path, **filters)
def load_reports_and_totals(committee_id, cycle, cycle_out_of_range, fallback_cycle):
filters = {
"committee_id": committee_id,
"cycle": fallback_cycle if cycle_out_of_range else cycle,
"per_page": 1,
"sort_hide_null": True,
}
# (3) call /filings? under tag:filings
# get reports from filings endpoint filter by form_category=REPORT
path = "/filings/"
reports = api_caller.load_first_row_data(
path, form_category="REPORT", most_recent=True, **filters
)
# (4)call committee/{committee_id}/totals? under tag:financial
# get financial totals
path = "/committee/" + committee_id + "/totals/"
totals = api_caller.load_first_row_data(path, **filters)
return reports, totals
def get_committee(committee_id, cycle):
"""
Given a committee_id and cycle, call the API and get the committee
and committee financial data needed to render the committee profile page
"""
committee, all_candidates, cycle = load_committee_history(
committee_id, cycle)
# When there are multiple candidate records of various offices (H, S, P)
# linked to a single committee ID,
# associate the candidate record with the matching committee type
candidates = [
candidate for candidate in all_candidates
if committee["committee_type"] == candidate["office"]
]
parent = "data"
result_type = "committees"
cycle = int(cycle)
year = to_date(committee, cycle)
# Link to current cycle if candidate has a corresponding page, else link
# without cycle query parameter
# See https://github.com/fecgov/openFEC/issues/1536
for candidate in candidates:
election_years = []
for election_year in candidate["election_years"]:
start_of_election_period = (
election_year - election_durations[candidate["office"]])
if start_of_election_period < cycle and cycle <= election_year:
election_years.append(election_year)
# For each candidate, set related_cycle to the candidate's time period
# relative to the selected cycle.
candidate["related_cycle"] = cycle if election_years else None
report_type = report_types.get(committee["committee_type"], "pac-party")
cycle_out_of_range, fallback_cycle, cycles = load_cycle_data(
committee, cycle)
reports, totals = load_reports_and_totals(committee_id, cycle,
cycle_out_of_range,
fallback_cycle)
# Check organization types to determine SSF status
is_ssf = committee.get("organization_type") in [
"W", "C", "L", "V", "M", "T"
]
# if cycles_has_activity's options are more than cycles_has_financial's,
# when clicking back to financial summary/raising/spending,
# reset cycle=fallback_cycle and timePeriod and.
# to make sure missing message page show correct timePeriod.
time_period_js = str(int(cycle) - 1) + "–" + str(cycle)
cycle_js = cycle
if cycle_out_of_range:
cycle_js = fallback_cycle
time_period_js = str(int(cycle_js) - 1) + "–" + str(cycle_js)
context_vars = {
"cycle": cycle_js,
"timePeriod": time_period_js,
"name": committee["name"],
"cycleOutOfRange": cycle_out_of_range,
"lastCycleHasFinancial": fallback_cycle,
}
template_variables = {
"candidates": candidates,
"committee": committee,
"committee_id": committee_id,
"committee_type": committee["committee_type"],
"context_vars": context_vars,
"cycle": cycle,
"cycles": cycles,
"is_SSF": is_ssf,
"cycle_out_of_range": cycle_out_of_range,
"parent": parent,
"result_type": result_type,
"report_type": report_type,
"reports": reports,
"totals": totals,
"min_receipt_date": utils.three_days_ago(),
"context_vars": context_vars,
"party_full": committee["party_full"],
"candidates": candidates,
"social_image_identifier": "data",
"year": year,
"timePeriod": time_period_js,
}
# Format the current two-year-period's totals
if reports and totals:
# IE-only committees
if committee["committee_type"] == "I":
template_variables["ie_summary"] = utils.process_ie_data(totals)
# Inaugural Committees
elif committee["organization_type"] == "I":
template_variables[
"inaugural_summary"] = utils.process_inaugural_data(totals)
# Host Committees that file on Form 4
elif committee["organization_type"] == "H" and reports[
"form_type"] == "F4":
template_variables[
"raising_summary"] = utils.process_host_raising_data(totals)
template_variables[
"spending_summary"] = utils.process_host_spending_data(totals)
template_variables["cash_summary"] = utils.process_cash_data(
totals)
else:
# All other committees have three tables
template_variables["raising_summary"] = utils.process_raising_data(
totals)
template_variables[
"spending_summary"] = utils.process_spending_data(totals)
template_variables["cash_summary"] = utils.process_cash_data(
totals)
# If in the current cycle, check for raw filings in the last three days
if cycle == utils.current_cycle():
# (4)call efile/filings under tag: efiling
path = "/efile/filings/"
filters = {}
filters["cycle"] = cycle
filters["committee_id"] = committee["committee_id"]
filters["min_receipt_date"] = template_variables["min_receipt_date"]
raw_filings = api_caller.load_endpoint_results(path, **filters)
template_variables["has_raw_filings"] = True if raw_filings else False
else:
template_variables["has_raw_filings"] = False
# Needed for filings tab
template_variables["filings_lookup"] = {
"reports": ["F3", "F3X", "F3P", "F3L", "F4", "F5", "F7", "F13"],
"notices": ["F5", "F24", "F6", "F9", "F10", "F11"],
"statements": ["F1"],
"other": ["F1M", "F8", "F99", "F12"],
}
# Call /filings?committee_id=C00693234&form_type=F1
# Get the statements of organization
statement_of_organization = api_caller.load_committee_statement_of_organization(
committee_id)
if statement_of_organization:
statement_of_organization["receipt_date"] = format_receipt_date(
statement_of_organization["receipt_date"])
template_variables["statement_of_organization"] = statement_of_organization
# Add message for a committee that was formerly an authorized candidate committee.
# These committees are now unauthorized committees.
converted_committee_id = None
former_committee_name = None
former_authorized_candidate_id = None
former_authorized_candidate_name = None
if cycle == 2020 and committee_to_candidate_linkage.get(committee_id):
# Call /candidate/{candidate_id}/history/
converted_committee_id = committee.get('committee_id')
former_authorized_candidate_id = committee_to_candidate_linkage.get(
converted_committee_id)
path = "/candidate/" + str(
former_authorized_candidate_id) + "/history/"
filters = {}
filters["per_page"] = 1
candidate = api_caller.load_first_row_data(path, **filters)
# Get the converted committee's former name and candidate name
former_committee_name = former_committee_names.get(
converted_committee_id)
former_authorized_candidate_name = candidate.get('name')
template_variables["former_committee_name"] = former_committee_name
template_variables[
"former_authorized_candidate_name"] = former_authorized_candidate_name
template_variables[
"former_authorized_candidate_id"] = former_authorized_candidate_id
return template_variables
def get_candidate(candidate_id, cycle, election_full):
"""
1) By passing parameter "candidate_id" to get candidate data.
2) "cycle" and "election_full" are optional parameters.
if no election_full, election_full = true.
if no cycle, cycle = the latest item in rounded_election_years.
3) totally call 5-6 endpoints.
"""
# (1)call candidate/{candidate_id}/history/ under tag:candidate
# get rounded_election_years(candidate_election_year).
path = "/candidate/" + candidate_id + "/history/"
filters = {}
filters["per_page"] = 1
candidate = api_caller.load_first_row_data(path, **filters)
if candidate is None:
raise Http404()
# a)Build List: even_election_years for candidate election dropdown list
# on candidate profile page.
# b)rounded_election_years round up any odd year special election_year
# to even number. (ex:2017=>2018)
# even_election_years = candidate.get('rounded_election_years')
# if cycle = null, set cycle = the last of rounded_election_years.
if not cycle:
cycle = max(candidate.get("rounded_election_years"))
# (2)call candidate/{candidate_id}/history/{cycle} under tag:candidate
# (3)call candidate/{candidate_id}/committees/history/{cycle}
# under tag:committee.
candidate, committees, cycle = api_caller.load_with_nested(
"candidate",
candidate_id,
"committees",
cycle=cycle,
election_full=election_full,
)
result_type = "candidates"
duration = election_durations.get(candidate["office"], 2)
min_cycle = cycle - duration if election_full else cycle
report_type = report_types.get(candidate["office"])
# For JavaScript
context_vars = {
"cycles": candidate["fec_cycles_in_election"],
"name": candidate["name"],
"cycle": cycle,
"electionFull": election_full,
"candidateID": candidate["candidate_id"],
}
max_cycle = int(cycle)
# Check if cycle > latest_fec_cycles_in_election, such as:future candidate.
if candidate["fec_cycles_in_election"]:
latest_fec_cycles_in_election = max(
candidate["fec_cycles_in_election"])
if int(cycle) > latest_fec_cycles_in_election:
max_cycle = latest_fec_cycles_in_election
# Annotate committees with most recent available cycle
aggregate_cycles = (list(range(cycle, cycle - duration, -2))
if election_full else [cycle])
for committee in committees:
committee["related_cycle"] = committee["cycle"]
# Group the committees by designation
committee_groups = groupby(committees, lambda each: each["designation"])
committees_authorized = committee_groups.get(
"P", []) + committee_groups.get("A", [])
committee_ids = [
committee["committee_id"] for committee in committees_authorized
]
# (4)call candidate/{candidate_id}/totals/{cycle} under tag:candidate
# Get aggregate totals for the financial summary
filters["election_full"] = election_full
filters["cycle"] = cycle
path = "/candidate/" + candidate_id + "/totals/"
aggregate = api_caller.load_first_row_data(path, **filters)
if election_full:
# (5)if election_full is true, need call
# candidate/{candidate_id}/totals/{cycle} second time
# for showing on raising and spending tabs
# Get most recent 2-year period totals
filters["election_full"] = False
filters["cycle"] = max_cycle
two_year_totals = api_caller.load_first_row_data(path, **filters)
else:
two_year_totals = aggregate
if aggregate:
raising_summary = utils.process_raising_data(aggregate)
spending_summary = utils.process_spending_data(aggregate)
cash_summary = utils.process_cash_data(aggregate)
else:
raising_summary = None
spending_summary = None
cash_summary = None
# (6)Call /filings?candidate_id=P00003392&form_type=F2
# Get the statements of candidacy
statement_of_candidacy = api_caller.load_candidate_statement_of_candidacy(
candidate["candidate_id"])
if statement_of_candidacy:
statement_of_candidacy["receipt_date"] = format_receipt_date(
statement_of_candidacy["receipt_date"])
# Get all the elections
elections = sorted(
zip(candidate["election_years"], candidate["election_districts"]),
key=lambda pair: pair[0],
reverse=True,
)
# (7)call efile/filings/ under tag:efiling
# Check if there are raw_filings for this candidate
raw_filing_start_date = utils.three_days_ago()
filters["min_receipt_date"] = raw_filing_start_date
filters["committee_id"] = candidate["candidate_id"]
filters["cycle"] = cycle
filters["per_page"] = 100
path = "/efile/" + "/filings/"
raw_filings = api_caller.load_endpoint_results(path, **filters)
has_raw_filings = True if raw_filings else False
# Add message for when a candidate converts their candidate committee to an unauthorized committee
current_committee_name = None
converted_committee_id = None
former_committee_name = None
if cycle == 2020 and candidate_to_committee_linkage.get(candidate_id):
# Call committee/{committee_id}/history/{cycle}/
filters = {"per_page": 1}
path = "/committee/" + candidate_to_committee_linkage.get(
candidate_id) + "/history/" + str(cycle)
committee = api_caller.load_first_row_data(path, **filters)
# Get the converted committee's name, committee ID, and former committee name
current_committee_name = committee.get('name')
converted_committee_id = committee.get('committee_id')
former_committee_name = former_committee_names.get(
converted_committee_id)
return {
"converted_committee_name": current_committee_name,
"converted_committee_id": converted_committee_id,
"former_committee_name": former_committee_name,
"aggregate": aggregate,
"aggregate_cycles": aggregate_cycles,
"candidate": candidate,
"candidate_id": candidate_id,
"cash_summary": cash_summary,
"committee_groups": committee_groups,
"committee_ids": committee_ids,
# filings endpoint takes candidate ID value as committee ID arg
"committee_id": candidate["candidate_id"],
"committees_authorized": committees_authorized,
"context_vars": context_vars,
"cycle": int(cycle),
"cycles": candidate["fec_cycles_in_election"],
"district": candidate["district"],
"duration": duration,
"election_year": cycle,
"election_years": candidate.get("rounded_election_years"),
"elections": elections,
"has_raw_filings": has_raw_filings,
"incumbent_challenge_full": candidate["incumbent_challenge_full"],
"max_cycle": max_cycle,
"min_cycle": min_cycle,
"min_receipt_date": raw_filing_start_date,
"name": candidate["name"],
"office": candidate["office"],
"office_full": candidate["office_full"],
"party_full": candidate["party_full"],
"raising_summary": raising_summary,
"report_type": report_type,
"result_type": result_type,
"show_full_election": election_full,
"spending_summary": spending_summary,
"state": candidate["state"],
"statement_of_candidacy": statement_of_candidacy,
"two_year_totals": two_year_totals,
"social_image_identifier": "data",
}
def get_candidate(candidate_id, cycle, election_full):
"""
1) By passing parameter "candidate_id" to get candidate data.
2) "cycle" and "election_full" are optional parameters.
if no election_full, election_full = true.
if no cycle, cycle = the latest item in rounded_election_years.
3) totally call 5-6 endpoints.
"""
# (1) Call candidate/{candidate_id}/history/ under tag:candidate
# get rounded_election_years(candidate_election_year).
candidate = load_most_recent_candidate(candidate_id)
if candidate is None:
raise Http404()
# if cycle = null, set cycle = the last of rounded_election_years.
if not cycle:
cycle = max(candidate.get("rounded_election_years"))
# (2) Call candidate/{candidate_id}/history/{cycle} under tag:candidate
# (3) Call candidate/{candidate_id}/committees/history/{cycle}
# under tag:committee.
candidate, committees, cycle = api_caller.load_with_nested(
"candidate",
candidate_id,
"committees",
cycle=cycle,
election_full=election_full,
)
result_type = "candidates"
duration = election_durations.get(candidate["office"], 2)
min_cycle = cycle - duration if election_full else cycle
report_type = report_types.get(candidate["office"])
# For JavaScript
context_vars = {
"cycles": candidate["fec_cycles_in_election"],
"name": candidate["name"],
"cycle": cycle,
"electionFull": election_full,
"candidateID": candidate["candidate_id"],
}
max_cycle = int(cycle)
# Check if cycle > latest_fec_cycles_in_election, such as:future candidate.
if candidate["fec_cycles_in_election"]:
latest_fec_cycles_in_election = max(
candidate["fec_cycles_in_election"])
if int(cycle) > latest_fec_cycles_in_election:
max_cycle = latest_fec_cycles_in_election
# Annotate committees with most recent available cycle
aggregate_cycles = (list(range(cycle, cycle - duration, -2))
if election_full else [cycle])
for committee in committees:
committee["related_cycle"] = committee["cycle"]
# Group the committees by designation
committee_groups = groupby(committees, lambda each: each["designation"])
committees_authorized = committee_groups.get(
"P", []) + committee_groups.get("A", [])
committee_ids = [
committee["committee_id"] for committee in committees_authorized
]
# Group the committees by leadership pac (designation=D)
committees_d = committee_groups.get("D", [])
committees_leadership_pac = []
if committees_d:
# The candidate(id=P00009183) returns two rows from api result,
# one is pcc converted to D, another is leadership pac committee,
# remove the duplicate committees with same committees_id.
# example api call:
# https://fec-dev-api.app.cloud.gov/v1/candidate/P00009183/committees/history/2020/
len_d_cmte = len(committees_d)
committees_leadership_pac.append(committees_d[0])
for i in range(len_d_cmte):
if (i + 1) < len_d_cmte and committees_d[i].get(
"committee_id") != committees_d[i + 1].get("committee_id"):
committees_leadership_pac.append(committees_d[i + 1])
# (4) Call candidate/{candidate_id}/totals/{cycle} under tag:candidate
# Get aggregate totals for the financial summary
filters = {"election_full": election_full, "cycle": cycle}
path = "/candidate/" + candidate_id + "/totals/"
aggregate = api_caller.load_first_row_data(path, **filters)
if election_full:
# (5) if election_full is true, need call
# candidate/{candidate_id}/totals/{cycle} second time
# for showing on raising and spending tabs
# Get most recent 2-year period totals
filters = {"election_full": False, "cycle": max_cycle}
two_year_totals = api_caller.load_first_row_data(path, **filters)
else:
two_year_totals = aggregate
if aggregate:
raising_summary = utils.process_raising_data(aggregate)
spending_summary = utils.process_spending_data(aggregate)
cash_summary = utils.process_cash_data(aggregate)
else:
raising_summary = None
spending_summary = None
cash_summary = None
# (6) Call /filings?candidate_id=P00003392&form_type=F2
# Get the statements of candidacy
statement_of_candidacy = api_caller.load_candidate_statement_of_candidacy(
candidate["candidate_id"])
if statement_of_candidacy:
statement_of_candidacy["receipt_date"] = format_receipt_date(
statement_of_candidacy["receipt_date"])
# Get all the elections
elections = sorted(
zip(candidate["election_years"], candidate["election_districts"]),
key=lambda pair: pair[0],
reverse=True,
)
# (7) Call efile/filings/ under tag:efiling
# Check if there are raw_filings for this candidate
raw_filing_start_date = utils.three_days_ago()
filters = {
"min_receipt_date": raw_filing_start_date,
"committee_id": candidate["candidate_id"],
"cycle": cycle,
"per_page": 100,
}
path = "/efile/" + "/filings/"
raw_filings = api_caller.load_endpoint_results(path, **filters)
has_raw_filings = True if raw_filings else False
# Add message for when a candidate converts their candidate committee to an unauthorized committee
current_committee_name = None
converted_committee_id = None
former_committee_name = None
# Get the latest committee name, former authorized committee name, and committee ID.
# This will be the first item returned in the committees list
# If there are no committees, return the normal no results message
if len(committees) > 0:
if committees[0].get('former_candidate_id'):
current_committee_name = committees[0].get('name')
converted_committee_id = committees[0].get('committee_id')
former_committee_name = committees[0].get('former_committee_name')
return {
"converted_committee_name": current_committee_name,
"converted_committee_id": converted_committee_id,
"former_committee_name": former_committee_name,
"aggregate": aggregate,
"aggregate_cycles": aggregate_cycles,
"candidate": candidate,
"candidate_id": candidate_id,
"cash_summary": cash_summary,
"committee_groups": committee_groups,
"committee_ids": committee_ids,
# filings endpoint takes candidate ID value as committee ID arg
"committee_id": candidate["candidate_id"],
"committees_authorized": committees_authorized,
"committees_leadership_pac": committees_leadership_pac,
"context_vars": context_vars,
"cycle": int(cycle),
"cycles": candidate["fec_cycles_in_election"],
"district": candidate["district"],
"duration": duration,
"election_year": cycle,
"election_years": candidate.get("rounded_election_years"),
"elections": elections,
"has_raw_filings": has_raw_filings,
"incumbent_challenge_full": candidate["incumbent_challenge_full"],
"max_cycle": max_cycle,
"min_cycle": min_cycle,
"min_receipt_date": raw_filing_start_date,
"name": candidate["name"],
"office": candidate["office"],
"office_full": candidate["office_full"],
"party_full": candidate["party_full"],
"raising_summary": raising_summary,
"report_type": report_type,
"result_type": result_type,
"show_full_election": election_full,
"spending_summary": spending_summary,
"state": candidate["state"],
"statement_of_candidacy": statement_of_candidacy,
"two_year_totals": two_year_totals,
"social_image_identifier": "data",
}
