def update_committee_from_masterfile(committee_id, cycle_to_copy_from=2014, cycle_to_copy_to=2014):
#print "Updating %s" % (committee_id)
c = None
try:
c = Committee.objects.get(cmte_id=committee_id, cycle=cycle_to_copy_from)
except Committee.MultipleObjectsReturned:
print "Multiple committees found with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from)
return None
except Committee.DoesNotExist:
print "Missing committee with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from)
return None
committee_overlay = None
try:
committee_overlay = Committee_Overlay.objects.get(fec_id=committee_id, cycle=cycle_to_copy_from)
except Committee_Overlay.DoesNotExist:
# This shouldn't happen
return None
ctype = c.cmte_tp
is_hybrid = False
is_noncommittee = False
is_superpac = False
if ctype:
if ctype.upper() in ['O', 'U']:
is_superpac = True
if ctype.upper() in ['V', 'W']:
is_hybrid = True
if ctype.upper() in ['I']:
is_noncommittee = True
party = c.cmte_pty_affiliation
if party:
party = get_party_from_pty(party)
committee_overlay.cycle = cycle_to_copy_to
committee_overlay.name = c.cmte_name
committee_overlay.slug = slugify(c.cmte_name)
committee_overlay.party = party
committee_overlay.treasurer = c.tres_nm
committee_overlay.street_1 = c.cmte_st1
committee_overlay.street_2 = c.cmte_st2
committee_overlay.city = c.cmte_city
committee_overlay.state = c.cmte_st
committee_overlay.connected_org_name = c.connected_org_nm
committee_overlay.filing_frequency = c.cmte_filing_freq
committee_overlay.candidate_id = c.cand_id
committee_overlay.is_superpac = is_superpac
committee_overlay.is_hybrid = is_hybrid
committee_overlay.is_noncommittee = is_noncommittee
committee_overlay.designation = c.cmte_dsgn
committee_overlay.ctype = ctype
committee_overlay.save()
def update_committee_from_masterfile(committee_id, cycle_to_copy_from=2014, cycle_to_copy_to=2014):
#print "Updating %s" % (committee_id)
c = None
try:
c = Committee.objects.get(cmte_id=committee_id, cycle=cycle_to_copy_from)
except Committee.MultipleObjectsReturned:
print "Multiple committees found with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from)
return None
except Committee.DoesNotExist:
print "Missing committee with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from)
return None
committee_overlay = None
try:
committee_overlay = Committee_Overlay.objects.get(fec_id=committee_id, cycle=cycle_to_copy_from)
except Committee_Overlay.DoesNotExist:
# This shouldn't happen
return None
ctype = c.cmte_tp
is_hybrid = False
is_noncommittee = False
is_superpac = False
if ctype:
if ctype.upper() in ['O', 'U']:
is_superpac = True
if ctype.upper() in ['V', 'W']:
is_hybrid = True
if ctype.upper() in ['I']:
is_noncommittee = True
party = c.cmte_pty_affiliation
if party:
party = get_party_from_pty(party)
committee_overlay.cycle = cycle_to_copy_to
committee_overlay.name = c.cmte_name
committee_overlay.slug = slugify(c.cmte_name)
committee_overlay.party = party
committee_overlay.treasurer = c.tres_nm
committee_overlay.street_1 = c.cmte_st1
committee_overlay.street_2 = c.cmte_st2
committee_overlay.city = c.cmte_city
committee_overlay.state = c.cmte_st
committee_overlay.connected_org_name = c.connected_org_nm
committee_overlay.filing_frequency = c.cmte_filing_freq
committee_overlay.candidate_id = c.cand_id
committee_overlay.is_superpac = is_superpac
committee_overlay.is_hybrid = is_hybrid
committee_overlay.is_noncommittee = is_noncommittee
committee_overlay.designation = c.cmte_dsgn
committee_overlay.ctype = ctype
committee_overlay.save()
def get_senate_challengers():
# then get the candidates who are declared. While we set the incumbents' election year as 2014--overriding whatever they have in their FEC filings, we don't do that here.
allSenateCandidates = Candidate.objects.filter(cycle=cycle, cand_office='S', cand_election_year__gte=2013)
for thiscandidate in allSenateCandidates:
fec_id = thiscandidate.cand_id
try:
senate_exclusion_hash[fec_id]
continue
except KeyError:
pass
#print "handling %s" % (thiscandidate.cand_name)
# do we already have this candidate entered ?
try:
entered_candidate = Candidate_Overlay.objects.get(election_year__gte=2013, fec_id=fec_id)
#print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici)
except Candidate_Overlay.DoesNotExist:
# don't enter if they say they're an incumbent. Problaby an error.
if thiscandidate.cand_ici == 'I':
print "!! Not-yet-entered Senate candidate %s %s %s claims incumbency. Perhaps they've retired?" % (thiscandidate.cand_name, thiscandidate.cand_election_year, thiscandidate.cand_office_st)
else:
# It's a new candidate. Enter it.
state = thiscandidate.cand_office_st
term_class = get_term_class_from_election_year(thiscandidate.cand_election_year)
if thiscandidate.cand_election_year < 2013:
print "!! Disregarding old candidate %s %s %s" % (thiscandidate.cand_name, thiscandidate.cand_election_year, state)
election_year = str(thiscandidate.cand_election_year)
try:
this_district = District.objects.get(election_year=election_year, state=state, office='S')
# Only make it an open seat if we've already marked the district as open.
cand_ici = 'C'
if this_district.open_seat:
cand_ici='O'
Candidate_Overlay.objects.create(
district=this_district,
cycle=cycle,
fec_id=fec_id,
name=thiscandidate.cand_name,
slug = slugify(thiscandidate.cand_name),
pty=thiscandidate.cand_pty_affiliation,
party = get_party_from_pty(thiscandidate.cand_pty_affiliation),
#pcc=thiscandidate.cand_pcc,
term_class=term_class,
election_year=thiscandidate.cand_election_year,
state=thiscandidate.cand_office_st,
office='S',
cand_ici=cand_ici,
candidate_status='D'
)
except District.DoesNotExist:
print "!! Invalid senate district for %s %s %s %s" % (thiscandidate.cand_name, term_class, thiscandidate.cand_election_year, state)
continue
def get_senate_incumbents():
# Get the incumbents
for member in senate_crosswalk:
bio = member['bioguide']
fec_id = member['fec_id']
if not fec_id:
# ignore
continue
#print "handling %s %s" % (bio, fec_id)
# term class isn't kept in the fec records...
thisterm = Term.objects.get(start__lte=cycle_end, end__gte=cycle_start, term_type='sen', legislator__bioguide=bio)
thiscandidate = Candidate.objects.get(cand_id=fec_id, cycle=cycle)
office = thiscandidate.cand_office
term_class = thisterm.term_class
state = thiscandidate.cand_office_st
#print "processing %s: %s %s %s" % (thiscandidate.cand_name, office, state, term_class)
election_year = get_election_year_from_term_class(term_class)
senate_cycle = str(election_year)
this_district = District.objects.get(cycle=senate_cycle,state=state, office='S', term_class=term_class)
this_overlay, created = Candidate_Overlay.objects.get_or_create(fec_id=fec_id, cycle=senate_cycle, district=this_district)
this_overlay.name = thiscandidate.cand_name
this_overlay.slug = slugify(thiscandidate.cand_name)
this_overlay.candidate = thiscandidate
this_overlay.office='S'
this_overlay.state = thiscandidate.cand_office_st
this_overlay.pty = thiscandidate.cand_pty_affiliation
this_overlay.party = get_party_from_pty(thiscandidate.cand_pty_affiliation)
this_overlay.pcc = thiscandidate.cand_pcc
this_overlay.cand_ici = 'I'
this_overlay.election_year = election_year
this_overlay.term_class=term_class
this_overlay.save()
try:
senate_casualty_hash[fec_id]
this_overlay.not_seeking_reelection = True
this_overlay.save()
this_district.open_seat = True
this_district.save()
except KeyError:
pass
def get_house_incumbents():
# Get the incumbents
for member in house_crosswalk:
bio = member['bioguide']
fec_id = member['fec_id']
thiscandidate = Candidate.objects.get(cand_id=fec_id, cycle=cycle)
#office = thiscandidate.cand_office
office = 'H'
district = None
district = thiscandidate.cand_office_district.zfill(2)
state = thiscandidate.cand_office_st
#print "processing %s: %s %s %s" % (thiscandidate.cand_name, office, state, district)
this_district = District.objects.get(cycle=cycle,state=state, office='H', office_district=district)
this_overlay, created = Candidate_Overlay.objects.get_or_create(fec_id=fec_id, cycle=cycle, district=this_district)
this_overlay.name = thiscandidate.cand_name
this_overlay.slug = slugify(thiscandidate.cand_name)
this_overlay.pty = thiscandidate.cand_pty_affiliation
this_overlay.party = get_party_from_pty(thiscandidate.cand_pty_affiliation)
this_overlay.pcc = thiscandidate.cand_pcc
this_overlay.cand_ici = 'I'
this_overlay.office = office
this_overlay.election_year = 2014
this_overlay.save()
try:
house_casualty_hash[fec_id]
this_overlay.not_seeking_reelection = True
this_overlay.save()
this_district.open_seat = True
this_district.save()
except KeyError:
pass
def make_candidate_overlay_from_masterfile(candidate_id, cycle_to_copy_from=2014, election_year=2014, cycle_to_copy_to=2014, verify_does_not_exist=True, display_candidate=False):
## Returns overlay if created, None if not.
if verify_does_not_exist:
try:
# If there's already a candidate overlay, don't do this.
entered_candidate = Candidate_Overlay.objects.get(cycle=cycle_to_copy_to, fec_id=candidate_id)
#print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici)
return None
except Candidate_Overlay.DoesNotExist:
pass
thiscandidate = None
try:
thiscandidate = Candidate.objects.get(cycle=cycle_to_copy_from, cand_id=candidate_id, cand_election_year=election_year)
except Candidate.DoesNotExist:
print "Couldn't find candidate in masterfile id=%s election_year=%s cycle=%s" % (candidate_id, election_year, cycle_to_copy_from)
return None
state = thiscandidate.cand_office_st
term_class = None
if thiscandidate.cand_office == 'S':
term_class = get_term_class_from_election_year(thiscandidate.cand_election_year)
this_district = None
try:
if thiscandidate.cand_office == 'S':
this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, term_class=term_class)
elif thiscandidate.cand_office == 'H':
this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, office_district=thiscandidate.cand_office_district)
elif thiscandidate.cand_office == 'P':
# there's no district, so pass
# do we need a presidential placeholder ?
pass
except District.DoesNotExist:
print "!! Invalid district for %s %s %s %s" % (thiscandidate.cand_name, term_class, thiscandidate.cand_election_year, state)
# If we can't find a district, override the display setting--just don't display it.
display_candidate = False
co = Candidate_Overlay.objects.create(
district=this_district,
office_district=thiscandidate.cand_office_district,
cycle=cycle_to_copy_to,
fec_id=candidate_id,
name=thiscandidate.cand_name,
slug = slugify(thiscandidate.cand_name),
pty=thiscandidate.cand_pty_affiliation,
party = get_party_from_pty(thiscandidate.cand_pty_affiliation),
pcc=thiscandidate.cand_pcc,
term_class=term_class,
election_year=thiscandidate.cand_election_year,
curated_election_year=thiscandidate.cand_election_year,
state=thiscandidate.cand_office_st,
office=thiscandidate.cand_office,
cand_ici=thiscandidate.cand_ici,
candidate_status='D',
display = display_candidate,
)
return co
def make_committee_overlay_from_masterfile(committee_id, cycle_to_copy_from=2014, cycle_to_copy_to=2014, verify_does_not_exist=True):
c = None
try:
c = Committee.objects.get(cmte_id=committee_id, cycle=cycle_to_copy_from)
except Committee.MultipleObjectsReturned:
print "Multiple committees found with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from)
return None
if verify_does_not_exist:
try:
Committee_Overlay.objects.get(fec_id=committee_id, cycle=cycle_to_copy_from)
# if it exists, update it with the current information
update_committee_from_masterfile(committee_id, cycle_to_copy_from=2014, cycle_to_copy_to=2014)
return None
except Committee_Overlay.DoesNotExist:
pass
ctype = c.cmte_tp
is_hybrid = False
is_noncommittee = False
is_superpac = False
is_paper_filer = False
if ctype:
if ctype.upper() in ['O', 'U']:
is_superpac = True
if ctype.upper() in ['V', 'W']:
is_hybrid = True
if ctype.upper() in ['I']:
is_noncommittee = True
if ctype.upper() in ['S']:
is_paper_filer = True
party = c.cmte_pty_affiliation
if party:
party = get_party_from_pty(party)
cm = Committee_Overlay.objects.create(
cycle = cycle_to_copy_to,
name = c.cmte_name,
fec_id = c.cmte_id,
slug = slugify(c.cmte_name),
party = party,
treasurer = c.tres_nm,
street_1 = c.cmte_st1,
street_2 = c.cmte_st2,
city = c.cmte_city,
state = c.cmte_st,
connected_org_name = c.connected_org_nm,
filing_frequency = c.cmte_filing_freq,
candidate_id = c.cand_id,
is_superpac = is_superpac,
is_hybrid = is_hybrid,
is_noncommittee = is_noncommittee,
designation = c.cmte_dsgn,
ctype = ctype,
is_dirty=True,
is_paper_filer=is_paper_filer
)
return cm
def handle_filing(this_filing):
try:
co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id,
cycle=this_filing.cycle)
this_filing.committee_designation = co.designation
this_filing.committee_name = co.name
this_filing.committee_type = co.ctype
this_filing.committee_slug = co.slug
this_filing.party = co.party
# mark that the committee is dirty
co.is_dirty = True
co.save()
except Committee_Overlay.DoesNotExist:
try:
## remember that ftpdata committees have cycles as ints, not strings. Not ideal.
if not this_filing.cycle:
this_filing.cycle = CURRENT_CYCLE
co = Committee.objects.get(cmte_id=this_filing.fec_id,
cycle=int(this_filing.cycle))
this_filing.committee_designation = co.cmte_dsgn
this_filing.committee_type = co.cmte_tp
this_filing.committee_name = co.cmte_name
this_filing.party = get_party_from_pty(co.cmte_pty_affiliation)
except Committee.DoesNotExist:
pass
header_data = this_filing.header_data
form_type = this_filing.form_type
parsed_data = {
'coh_start': None,
'coh_end': None,
'new_loans': None,
'tot_raised': None,
'tot_spent': None
}
if form_type in ['F3A', 'F3N', 'F3T', 'F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']:
parsed_data = process_f3_header(header_data)
print "got data %s" % (parsed_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data[
'coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data[
'new_loans'] else 0
this_filing.new_filing_details_set = True
elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']:
parsed_data = process_f3x_header(header_data)
print "got data %s" % (parsed_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data[
'coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data[
'new_loans'] else 0
this_filing.tot_coordinated = parsed_data[
'tot_coordinated'] if parsed_data['tot_coordinated'] else 0
this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data[
'tot_ies'] else 0
this_filing.new_filing_details_set = True
elif form_type in ['F5', 'F5A', 'F5N']:
parsed_data = process_f5_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
# total spending is total ies
this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date']
this_filing.coverage_to_date = parsed_data['coverage_to_date']
try:
this_filing.is_f5_quarterly = header_data['report_code'] in [
'Q1', 'Q2', 'Q3', 'Q4', 'YE'
]
except KeyError:
# this is probably a problem.
pass
this_filing.new_filing_details_set = True
elif form_type in ['F7', 'F7A', 'F7N']:
parsed_data = process_f7_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.coverage_from_date = parsed_data[
'coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data[
'coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.new_filing_details_set = True
elif form_type in ['F9', 'F9A', 'F9N']:
parsed_data = process_f9_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.coverage_from_date = parsed_data[
'coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data[
'coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.new_filing_details_set = True
elif form_type in ['F13', 'F13A', 'F13N']:
parsed_data = process_f13_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.coverage_from_date = parsed_data[
'coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data[
'coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.new_filing_details_set = True
else:
# Nothing to be done, but mark this step as done.
this_filing.new_filing_details_set = True
this_filing.save()
def make_candidate_overlay_from_masterfile(
candidate_id,
cycle_to_copy_from=2016,
election_year=2016,
cycle_to_copy_to=2016,
verify_does_not_exist=True,
display_candidate=False,
):
if candidate_id == "P20003851":
return None
## Returns overlay if created, None if not.
if verify_does_not_exist:
try:
# If there's already a candidate overlay, don't do this.
entered_candidate = Candidate_Overlay.objects.get(cycle=cycle_to_copy_to, fec_id=candidate_id)
# print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici)
return None
except Candidate_Overlay.DoesNotExist:
pass
thiscandidate = None
try:
thiscandidate = Candidate.objects.get(
cycle=cycle_to_copy_from, cand_id=candidate_id, cand_election_year=election_year
)
except Candidate.DoesNotExist:
print "Couldn't find candidate in masterfile id=%s election_year=%s cycle=%s" % (
candidate_id,
election_year,
cycle_to_copy_from,
)
return None
state = thiscandidate.cand_office_st
term_class = None
if thiscandidate.cand_office == "S":
term_class = get_term_class_from_election_year(thiscandidate.cand_election_year)
this_district = None
try:
if thiscandidate.cand_office == "S":
this_district = District.objects.get(
election_year=election_year, state=state, office=thiscandidate.cand_office, term_class=term_class
)
elif thiscandidate.cand_office == "H":
this_district = District.objects.get(
election_year=election_year,
state=state,
office=thiscandidate.cand_office,
office_district=thiscandidate.cand_office_district,
)
elif thiscandidate.cand_office == "P":
# there's a single presidential district per cycle, it needs to be created manually
this_district = District.objects.get(election_year=election_year, office=thiscandidate.cand_office)
except District.DoesNotExist:
print "!! Invalid %s district for %s term_class=%s district=%s election_year=%s state=%s" % (
thiscandidate.cand_office,
thiscandidate.cand_name,
term_class,
thiscandidate.cand_office_district,
thiscandidate.cand_election_year,
state,
)
# If we can't find a district, override the display setting--just don't display it.
display_candidate = False
co = Candidate_Overlay.objects.create(
district=this_district,
office_district=thiscandidate.cand_office_district,
cycle=cycle_to_copy_to,
fec_id=candidate_id,
name=thiscandidate.cand_name,
slug=slugify(thiscandidate.cand_name),
pty=thiscandidate.cand_pty_affiliation,
party=get_party_from_pty(thiscandidate.cand_pty_affiliation),
pcc=thiscandidate.cand_pcc,
term_class=term_class,
election_year=thiscandidate.cand_election_year,
curated_election_year=thiscandidate.cand_election_year,
state=thiscandidate.cand_office_st,
office=thiscandidate.cand_office,
cand_ici=thiscandidate.cand_ici,
candidate_status="D",
display=display_candidate,
)
return co
def make_committee_overlay_from_masterfile(
committee_id, cycle_to_copy_from=2016, cycle_to_copy_to=2016, verify_does_not_exist=True
):
if committee_id == "C00507947":
return None
c = None
try:
c = Committee.objects.get(cmte_id=committee_id, cycle=cycle_to_copy_from)
except Committee.MultipleObjectsReturned:
print "Multiple committees found with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from)
return None
if verify_does_not_exist:
try:
Committee_Overlay.objects.get(fec_id=committee_id, cycle=cycle_to_copy_from)
# if it exists, update it with the current information
update_committee_from_masterfile(committee_id, cycle_to_copy_from=2016, cycle_to_copy_to=2016)
return None
except Committee_Overlay.DoesNotExist:
pass
ctype = c.cmte_tp
is_hybrid = False
is_noncommittee = False
is_superpac = False
is_paper_filer = False
if ctype:
if ctype.upper() in ["O", "U"]:
is_superpac = True
if ctype.upper() in ["V", "W"]:
is_hybrid = True
if ctype.upper() in ["I"]:
is_noncommittee = True
if ctype.upper() in ["S"]:
is_paper_filer = True
party = c.cmte_pty_affiliation
if party:
party = get_party_from_pty(party)
cm = Committee_Overlay.objects.create(
cycle=cycle_to_copy_to,
name=c.cmte_name,
fec_id=c.cmte_id,
slug=slugify(c.cmte_name),
party=party,
treasurer=c.tres_nm,
street_1=c.cmte_st1,
street_2=c.cmte_st2,
city=c.cmte_city,
state=c.cmte_st,
connected_org_name=c.connected_org_nm,
filing_frequency=c.cmte_filing_freq,
candidate_id=c.cand_id,
is_superpac=is_superpac,
is_hybrid=is_hybrid,
is_noncommittee=is_noncommittee,
designation=c.cmte_dsgn,
ctype=ctype,
is_dirty=True,
is_paper_filer=is_paper_filer,
)
return cm
def handle_filing(this_filing):
try:
co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id)
this_filing.committee_designation = co.designation
this_filing.committee_type = co.ctype
this_filing.committee_slug = co.slug
this_filing.party = co.party
# mark that the committee is dirty
co.is_dirty = True
co.save()
except Committee_Overlay.DoesNotExist:
try:
co = Committee.objects.get(cmte_id=this_filing.fec_id, cycle=2014)
this_filing.committee_designation = co.cmte_dsgn
this_filing.committee_type = co.cmte_tp
this_filing.party = get_party_from_pty(co.cmte_pty_affiliation)
except Committee.DoesNotExist:
pass
header_data = this_filing.header_data
form_type = this_filing.form_type
parsed_data = {
'coh_start': None,
'coh_end': None,
'new_loans': None,
'tot_raised': None,
'tot_spent': None
}
if form_type in ['F3A', 'F3N', 'F3T', 'F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']:
parsed_data = process_f3_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data[
'coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data[
'new_loans'] else 0
elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']:
parsed_data = process_f3x_header(header_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data[
'coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data[
'new_loans'] else 0
this_filing.tot_coordinated = parsed_data[
'tot_coordinated'] if parsed_data['tot_coordinated'] else 0
this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data[
'tot_ies'] else 0
elif form_type in ['F5', 'F5A', 'F5N']:
parsed_data = process_f5_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
# total spending is total ies
this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date']
this_filing.coverage_to_date = parsed_data['coverage_to_date']
this_filing.is_f5_quarterly = header_data['report_code'] in [
'Q1', 'Q2', 'Q3', 'Q4', 'YE'
]
elif form_type in ['F7', 'F7A', 'F7N']:
parsed_data = process_f7_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.coverage_from_date = parsed_data[
'coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data[
'coverage_to_date'] if parsed_data['coverage_to_date'] else None
elif form_type in ['F9', 'F9A', 'F9N']:
parsed_data = process_f9_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[
'tot_spent'] else 0
this_filing.coverage_from_date = parsed_data[
'coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data[
'coverage_to_date'] if parsed_data['coverage_to_date'] else None
elif form_type in ['F13', 'F13A', 'F13N']:
parsed_data = process_f13_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[
'tot_raised'] else 0
this_filing.coverage_from_date = parsed_data[
'coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data[
'coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.save()
def handle_filing(this_filing):
try:
co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id)
this_filing.committee_designation = co.designation
this_filing.committee_type = co.ctype
this_filing.committee_slug = co.slug
this_filing.party = co.party
# mark that the committee is dirty
co.is_dirty=True
co.save()
except Committee_Overlay.DoesNotExist:
try:
co = Committee.objects.get(cmte_id=this_filing.fec_id, cycle=2014)
this_filing.committee_designation = co.cmte_dsgn
this_filing.committee_type = co.cmte_tp
this_filing.party = get_party_from_pty(co.cmte_pty_affiliation)
except Committee.DoesNotExist:
pass
header_data = this_filing.header_data
form_type = this_filing.form_type
parsed_data = {'coh_start':None, 'coh_end':None, 'new_loans':None,'tot_raised':None,'tot_spent':None}
if form_type in ['F3A', 'F3N', 'F3T','F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']:
parsed_data = process_f3_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0
elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']:
parsed_data = process_f3x_header(header_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0
this_filing.tot_coordinated = parsed_data['tot_coordinated'] if parsed_data['tot_coordinated'] else 0
this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data['tot_ies'] else 0
elif form_type in ['F5', 'F5A', 'F5N']:
parsed_data = process_f5_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
# total spending is total ies
this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date']
this_filing.coverage_to_date = parsed_data['coverage_to_date']
this_filing.is_f5_quarterly = header_data['report_code'] in ['Q1', 'Q2', 'Q3', 'Q4', 'YE']
elif form_type in ['F7', 'F7A', 'F7N']:
parsed_data = process_f7_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None
elif form_type in ['F9', 'F9A', 'F9N']:
parsed_data = process_f9_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None
elif form_type in ['F13', 'F13A', 'F13N']:
parsed_data = process_f13_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.save()
def make_candidate_overlay_from_masterfile(candidate_id, cycle_to_copy_from=2014, election_year=2014, cycle_to_copy_to=2014, verify_does_not_exist=True, display_candidate=False):
if candidate_id == 'P20003851':
return None
## Returns overlay if created, None if not.
if verify_does_not_exist:
try:
# If there's already a candidate overlay, don't do this.
entered_candidate = Candidate_Overlay.objects.get(cycle=cycle_to_copy_to, fec_id=candidate_id)
#print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici)
return None
except Candidate_Overlay.DoesNotExist:
pass
thiscandidate = None
try:
thiscandidate = Candidate.objects.get(cycle=cycle_to_copy_from, cand_id=candidate_id, cand_election_year=election_year)
except Candidate.DoesNotExist:
print "Couldn't find candidate in masterfile id=%s election_year=%s cycle=%s" % (candidate_id, election_year, cycle_to_copy_from)
return None
state = thiscandidate.cand_office_st
term_class = None
if thiscandidate.cand_office == 'S':
term_class = get_term_class_from_election_year(thiscandidate.cand_election_year)
this_district = None
try:
if thiscandidate.cand_office == 'S':
this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, term_class=term_class)
elif thiscandidate.cand_office == 'H':
this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, office_district=thiscandidate.cand_office_district)
elif thiscandidate.cand_office == 'P':
# there's no district, so pass
# do we need a presidential placeholder ?
pass
except District.DoesNotExist:
print "!! Invalid district for %s %s %s %s" % (thiscandidate.cand_name, term_class, thiscandidate.cand_election_year, state)
# If we can't find a district, override the display setting--just don't display it.
display_candidate = False
co = Candidate_Overlay.objects.create(
district=this_district,
office_district=thiscandidate.cand_office_district,
cycle=cycle_to_copy_to,
fec_id=candidate_id,
name=thiscandidate.cand_name,
slug = slugify(thiscandidate.cand_name),
pty=thiscandidate.cand_pty_affiliation,
party = get_party_from_pty(thiscandidate.cand_pty_affiliation),
pcc=thiscandidate.cand_pcc,
term_class=term_class,
election_year=thiscandidate.cand_election_year,
curated_election_year=thiscandidate.cand_election_year,
state=thiscandidate.cand_office_st,
office=thiscandidate.cand_office,
cand_ici=thiscandidate.cand_ici,
candidate_status='D',
display = display_candidate,
)
return co
def handle_filing(this_filing):
try:
co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id, cycle=this_filing.cycle)
this_filing.committee_designation = co.designation
this_filing.committee_name = co.name
this_filing.committee_type = co.ctype
this_filing.committee_slug = co.slug
this_filing.party = co.party
# mark that the committee is dirty
co.is_dirty=True
co.save()
except Committee_Overlay.DoesNotExist:
try:
## remember that ftpdata committees have cycles as ints, not strings. Not ideal.
if not this_filing.cycle:
this_filing.cycle = CURRENT_CYCLE
co = Committee.objects.get(cmte_id=this_filing.fec_id, cycle=int(this_filing.cycle))
this_filing.committee_designation = co.cmte_dsgn
this_filing.committee_type = co.cmte_tp
this_filing.committee_name = co.cmte_name
this_filing.party = get_party_from_pty(co.cmte_pty_affiliation)
except Committee.DoesNotExist:
pass
header_data = this_filing.header_data
form_type = this_filing.form_type
parsed_data = {'coh_start':None, 'coh_end':None, 'new_loans':None,'tot_raised':None,'tot_spent':None}
if form_type in ['F3A', 'F3N', 'F3T','F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']:
parsed_data = process_f3_header(header_data)
print "got data %s" % (parsed_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0
this_filing.new_filing_details_set = True
elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']:
parsed_data = process_f3x_header(header_data)
print "got data %s" % (parsed_data)
this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0
this_filing.tot_coordinated = parsed_data['tot_coordinated'] if parsed_data['tot_coordinated'] else 0
this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data['tot_ies'] else 0
this_filing.new_filing_details_set = True
elif form_type in ['F5', 'F5A', 'F5N']:
parsed_data = process_f5_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
# total spending is total ies
this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date']
this_filing.coverage_to_date = parsed_data['coverage_to_date']
try:
this_filing.is_f5_quarterly = header_data['report_code'] in ['Q1', 'Q2', 'Q3', 'Q4', 'YE']
except KeyError:
# this is probably a problem.
pass
this_filing.new_filing_details_set = True
elif form_type in ['F7', 'F7A', 'F7N']:
parsed_data = process_f7_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.new_filing_details_set = True
elif form_type in ['F9', 'F9A', 'F9N']:
parsed_data = process_f9_header(header_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.new_filing_details_set = True
elif form_type in ['F13', 'F13A', 'F13N']:
parsed_data = process_f13_header(header_data)
#print "got data %s" % (parsed_data)
this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0
this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None
this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None
this_filing.new_filing_details_set = True
else:
# Nothing to be done, but mark this step as done.
this_filing.new_filing_details_set = True
this_filing.save()
